1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.List;
29
30 import org.apache.commons.lang.math.RandomUtils;
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HColumnDescriptor;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.HTableDescriptor;
40 import org.apache.hadoop.hbase.testclassification.LargeTests;
41 import org.apache.hadoop.hbase.MiniHBaseCluster;
42 import org.apache.hadoop.hbase.ServerName;
43 import org.apache.hadoop.hbase.TableName;
44 import org.apache.hadoop.hbase.UnknownRegionException;
45 import org.apache.hadoop.hbase.MetaTableAccessor;
46 import org.apache.hadoop.hbase.client.Admin;
47 import org.apache.hadoop.hbase.client.Put;
48 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
49 import org.apache.hadoop.hbase.client.Result;
50 import org.apache.hadoop.hbase.client.ResultScanner;
51 import org.apache.hadoop.hbase.client.Scan;
52 import org.apache.hadoop.hbase.client.Table;
53 import org.apache.hadoop.hbase.exceptions.MergeRegionException;
54 import org.apache.hadoop.hbase.master.AssignmentManager;
55 import org.apache.hadoop.hbase.master.HMaster;
56 import org.apache.hadoop.hbase.master.RegionState.State;
57 import org.apache.hadoop.hbase.master.RegionStates;
58 import org.apache.hadoop.hbase.util.Bytes;
59 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
60 import org.apache.hadoop.hbase.util.FSUtils;
61 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
62 import org.apache.hadoop.hbase.util.Pair;
63 import org.apache.hadoop.hbase.util.PairOfSameType;
64 import org.apache.hadoop.util.StringUtils;
65 import org.junit.AfterClass;
66 import org.junit.BeforeClass;
67 import org.junit.Test;
68 import org.junit.experimental.categories.Category;
69
70 import com.google.common.base.Joiner;
71
72
73
74
75
76
77
78 @Category(LargeTests.class)
79 public class TestRegionMergeTransactionOnCluster {
80 private static final Log LOG = LogFactory
81 .getLog(TestRegionMergeTransactionOnCluster.class);
82 private static final int NB_SERVERS = 3;
83
84 private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
85 private static final byte[] QUALIFIER = Bytes.toBytes("q");
86
87 private static byte[] ROW = Bytes.toBytes("testRow");
88 private static final int INITIAL_REGION_NUM = 10;
89 private static final int ROWSIZE = 200;
90 private static byte[][] ROWS = makeN(ROW, ROWSIZE);
91
92 private static int waitTime = 60 * 1000;
93
94 static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
95
96 private static HMaster master;
97 private static Admin admin;
98
99 static void setupOnce() throws Exception {
100
101 TEST_UTIL.startMiniCluster(NB_SERVERS);
102 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
103 master = cluster.getMaster();
104 master.balanceSwitch(false);
105 admin = TEST_UTIL.getHBaseAdmin();
106 }
107
108 @BeforeClass
109 public static void beforeAllTests() throws Exception {
110
111 TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
112 setupOnce();
113 }
114
115 @AfterClass
116 public static void afterAllTests() throws Exception {
117 TEST_UTIL.shutdownMiniCluster();
118 }
119
120 @Test
121 public void testWholesomeMerge() throws Exception {
122 LOG.info("Starting testWholesomeMerge");
123 final TableName tableName =
124 TableName.valueOf("testWholesomeMerge");
125
126
127 Table table = createTableAndLoadData(master, tableName);
128
129 mergeRegionsAndVerifyRegionNum(master, tableName, 0, 1,
130 INITIAL_REGION_NUM - 1);
131
132
133 PairOfSameType<HRegionInfo> mergedRegions =
134 mergeRegionsAndVerifyRegionNum(master, tableName, 1, 2,
135 INITIAL_REGION_NUM - 2);
136
137 verifyRowCount(table, ROWSIZE);
138
139
140 HRegionInfo hri = RandomUtils.nextBoolean() ?
141 mergedRegions.getFirst() : mergedRegions.getSecond();
142 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
143 AssignmentManager am = cluster.getMaster().getAssignmentManager();
144 RegionStates regionStates = am.getRegionStates();
145 long start = EnvironmentEdgeManager.currentTime();
146 while (!regionStates.isRegionInState(hri, State.MERGED)) {
147 assertFalse("Timed out in waiting one merged region to be in state MERGED",
148 EnvironmentEdgeManager.currentTime() - start > 60000);
149 Thread.sleep(500);
150 }
151
152
153 am.assign(hri, true, true);
154 assertFalse("Merged region can't be assigned",
155 regionStates.isRegionInTransition(hri));
156 assertTrue(regionStates.isRegionInState(hri, State.MERGED));
157
158
159 am.unassign(hri, true, null);
160 assertFalse("Merged region can't be unassigned",
161 regionStates.isRegionInTransition(hri));
162 assertTrue(regionStates.isRegionInState(hri, State.MERGED));
163
164 table.close();
165 }
166
167 @Test
168 public void testCleanMergeReference() throws Exception {
169 LOG.info("Starting testCleanMergeReference");
170 admin.enableCatalogJanitor(false);
171 try {
172 final TableName tableName =
173 TableName.valueOf("testCleanMergeReference");
174
175 Table table = createTableAndLoadData(master, tableName);
176
177 mergeRegionsAndVerifyRegionNum(master, tableName, 0, 1,
178 INITIAL_REGION_NUM - 1);
179 verifyRowCount(table, ROWSIZE);
180 table.close();
181
182 List<Pair<HRegionInfo, ServerName>> tableRegions = MetaTableAccessor
183 .getTableRegionsAndLocations(master.getZooKeeper(), master.getConnection(), tableName);
184 HRegionInfo mergedRegionInfo = tableRegions.get(0).getFirst();
185 HTableDescriptor tableDescriptor = master.getTableDescriptors().get(
186 tableName);
187 Result mergedRegionResult = MetaTableAccessor.getRegionResult(
188 master.getConnection(), mergedRegionInfo.getRegionName());
189
190
191 assertTrue(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
192 HConstants.MERGEA_QUALIFIER) != null);
193 assertTrue(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
194 HConstants.MERGEB_QUALIFIER) != null);
195
196
197 HRegionInfo regionA = HRegionInfo.getHRegionInfo(mergedRegionResult,
198 HConstants.MERGEA_QUALIFIER);
199 HRegionInfo regionB = HRegionInfo.getHRegionInfo(mergedRegionResult,
200 HConstants.MERGEB_QUALIFIER);
201 FileSystem fs = master.getMasterFileSystem().getFileSystem();
202 Path rootDir = master.getMasterFileSystem().getRootDir();
203
204 Path tabledir = FSUtils.getTableDir(rootDir, mergedRegionInfo.getTable());
205 Path regionAdir = new Path(tabledir, regionA.getEncodedName());
206 Path regionBdir = new Path(tabledir, regionB.getEncodedName());
207 assertTrue(fs.exists(regionAdir));
208 assertTrue(fs.exists(regionBdir));
209
210 HColumnDescriptor[] columnFamilies = tableDescriptor.getColumnFamilies();
211 HRegionFileSystem hrfs = new HRegionFileSystem(
212 TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo);
213 int count = 0;
214 for(HColumnDescriptor colFamily : columnFamilies) {
215 count += hrfs.getStoreFiles(colFamily.getName()).size();
216 }
217 admin.compactRegion(mergedRegionInfo.getRegionName());
218
219
220 long timeout = System.currentTimeMillis() + waitTime;
221 int newcount = 0;
222 while (System.currentTimeMillis() < timeout) {
223 for(HColumnDescriptor colFamily : columnFamilies) {
224 newcount += hrfs.getStoreFiles(colFamily.getName()).size();
225 }
226 if(newcount > count) {
227 break;
228 }
229 Thread.sleep(50);
230 }
231 assertTrue(newcount > count);
232 List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
233 .getRegionServerThreads();
234 for (RegionServerThread rs : regionServerThreads) {
235 CompactedHFilesDischarger cleaner = new CompactedHFilesDischarger(100, null,
236 rs.getRegionServer(), false);
237 cleaner.chore();
238 Thread.sleep(1000);
239 }
240 int newcount1 = 0;
241 while (System.currentTimeMillis() < timeout) {
242 for(HColumnDescriptor colFamily : columnFamilies) {
243 newcount1 += hrfs.getStoreFiles(colFamily.getName()).size();
244 }
245 if(newcount1 <= 1) {
246 break;
247 }
248 Thread.sleep(50);
249 }
250
251
252 int cleaned = admin.runCatalogScan();
253 assertTrue(cleaned > 0);
254 assertFalse(fs.exists(regionAdir));
255 assertFalse(fs.exists(regionBdir));
256
257 mergedRegionResult = MetaTableAccessor.getRegionResult(
258 master.getConnection(), mergedRegionInfo.getRegionName());
259 assertFalse(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
260 HConstants.MERGEA_QUALIFIER) != null);
261 assertFalse(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
262 HConstants.MERGEB_QUALIFIER) != null);
263
264 } finally {
265 admin.enableCatalogJanitor(true);
266 }
267 }
268
269
270
271
272
273
274
275 @Test
276 public void testMerge() throws Exception {
277 LOG.info("Starting testMerge");
278 final TableName tableName = TableName.valueOf("testMerge");
279
280 try {
281
282 Table table = createTableAndLoadData(master, tableName);
283 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
284 List<HRegionInfo> regions = regionStates.getRegionsOfTable(tableName);
285
286 HRegionInfo a = regions.get(0);
287 HRegionInfo b = regions.get(1);
288 regionStates.regionOffline(a);
289 try {
290
291 admin.mergeRegions(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false);
292 fail("Offline regions should not be able to merge");
293 } catch (IOException ie) {
294 System.out.println(ie);
295 assertTrue("Exception should mention regions not online",
296 StringUtils.stringifyException(ie).contains("regions not online")
297 && ie instanceof MergeRegionException);
298 }
299 try {
300
301 admin.mergeRegions(b.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), true);
302 fail("A region should not be able to merge with itself, even forcifully");
303 } catch (IOException ie) {
304 assertTrue("Exception should mention regions not online",
305 StringUtils.stringifyException(ie).contains("region to itself")
306 && ie instanceof MergeRegionException);
307 }
308 try {
309
310 admin.mergeRegions(Bytes.toBytes("-f1"), Bytes.toBytes("-f2"), true);
311 fail("Unknown region could not be merged");
312 } catch (IOException ie) {
313 assertTrue("UnknownRegionException should be thrown",
314 ie instanceof UnknownRegionException);
315 }
316 table.close();
317 } finally {
318 TEST_UTIL.deleteTable(tableName);
319 }
320 }
321
322 @Test
323 public void testMergeWithReplicas() throws Exception {
324 final TableName tableName = TableName.valueOf("testMergeWithReplicas");
325
326 createTableAndLoadData(master, tableName, 5, 2);
327 List<Pair<HRegionInfo, ServerName>> initialRegionToServers =
328 MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(), master.getConnection(),
329 tableName);
330
331 PairOfSameType<HRegionInfo> mergedRegions = mergeRegionsAndVerifyRegionNum(master, tableName,
332 0, 2, 5 * 2 - 2);
333 List<Pair<HRegionInfo, ServerName>> currentRegionToServers =
334 MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(), master.getConnection(),
335 tableName);
336 List<HRegionInfo> initialRegions = new ArrayList<HRegionInfo>();
337 for (Pair<HRegionInfo, ServerName> p : initialRegionToServers) {
338 initialRegions.add(p.getFirst());
339 }
340 List<HRegionInfo> currentRegions = new ArrayList<HRegionInfo>();
341 for (Pair<HRegionInfo, ServerName> p : currentRegionToServers) {
342 currentRegions.add(p.getFirst());
343 }
344 assertTrue(initialRegions.contains(mergedRegions.getFirst()));
345 assertTrue(initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
346 mergedRegions.getFirst(), 1)));
347 assertTrue(initialRegions.contains(mergedRegions.getSecond()));
348 assertTrue(initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
349 mergedRegions.getSecond(), 1)));
350 assertTrue(!initialRegions.contains(currentRegions.get(0)));
351 assertTrue(!initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
352 currentRegions.get(0), 1)));
353 assertTrue(currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
354 currentRegions.get(0), 1)));
355 assertTrue(!currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
356 mergedRegions.getFirst(), 1)));
357 assertTrue(!currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
358 mergedRegions.getSecond(), 1)));
359 }
360
361 private PairOfSameType<HRegionInfo> mergeRegionsAndVerifyRegionNum(
362 HMaster master, TableName tablename,
363 int regionAnum, int regionBnum, int expectedRegionNum) throws Exception {
364 PairOfSameType<HRegionInfo> mergedRegions =
365 requestMergeRegion(master, tablename, regionAnum, regionBnum);
366 waitAndVerifyRegionNum(master, tablename, expectedRegionNum);
367 return mergedRegions;
368 }
369
370 private PairOfSameType<HRegionInfo> requestMergeRegion(
371 HMaster master, TableName tablename,
372 int regionAnum, int regionBnum) throws Exception {
373 List<Pair<HRegionInfo, ServerName>> tableRegions = MetaTableAccessor
374 .getTableRegionsAndLocations(master.getZooKeeper(),
375 master.getConnection(), tablename);
376 HRegionInfo regionA = tableRegions.get(regionAnum).getFirst();
377 HRegionInfo regionB = tableRegions.get(regionBnum).getFirst();
378 TEST_UTIL.getHBaseAdmin().mergeRegions(
379 regionA.getEncodedNameAsBytes(),
380 regionB.getEncodedNameAsBytes(), false);
381 return new PairOfSameType<HRegionInfo>(regionA, regionB);
382 }
383
384 private void waitAndVerifyRegionNum(HMaster master, TableName tablename,
385 int expectedRegionNum) throws Exception {
386 List<Pair<HRegionInfo, ServerName>> tableRegionsInMeta;
387 List<HRegionInfo> tableRegionsInMaster;
388 long timeout = System.currentTimeMillis() + waitTime;
389 while (System.currentTimeMillis() < timeout) {
390 tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(),
391 master.getConnection(), tablename);
392 tableRegionsInMaster = master.getAssignmentManager().getRegionStates()
393 .getRegionsOfTable(tablename);
394 if (tableRegionsInMeta.size() == expectedRegionNum
395 && tableRegionsInMaster.size() == expectedRegionNum) {
396 break;
397 }
398 Thread.sleep(250);
399 }
400
401 tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(),
402 master.getConnection(), tablename);
403 LOG.info("Regions after merge:" + Joiner.on(',').join(tableRegionsInMeta));
404 assertEquals(expectedRegionNum, tableRegionsInMeta.size());
405 }
406
407 private Table createTableAndLoadData(HMaster master, TableName tablename)
408 throws Exception {
409 return createTableAndLoadData(master, tablename, INITIAL_REGION_NUM, 1);
410 }
411
412 private Table createTableAndLoadData(HMaster master, TableName tablename,
413 int numRegions, int replication) throws Exception {
414 assertTrue("ROWSIZE must > numregions:" + numRegions, ROWSIZE > numRegions);
415 byte[][] splitRows = new byte[numRegions - 1][];
416 for (int i = 0; i < splitRows.length; i++) {
417 splitRows[i] = ROWS[(i + 1) * ROWSIZE / numRegions];
418 }
419
420 Table table = TEST_UTIL.createTable(tablename, FAMILYNAME, splitRows);
421 if (replication > 1) {
422 HBaseTestingUtility.setReplicas(admin, tablename, replication);
423 }
424 loadData(table);
425 verifyRowCount(table, ROWSIZE);
426
427
428 long timeout = System.currentTimeMillis() + waitTime;
429 List<Pair<HRegionInfo, ServerName>> tableRegions;
430 while (System.currentTimeMillis() < timeout) {
431 tableRegions = MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(),
432 master.getConnection(), tablename);
433 if (tableRegions.size() == numRegions * replication)
434 break;
435 Thread.sleep(250);
436 }
437
438 tableRegions = MetaTableAccessor.getTableRegionsAndLocations(
439 master.getZooKeeper(),
440 master.getConnection(), tablename);
441 LOG.info("Regions after load: " + Joiner.on(',').join(tableRegions));
442 assertEquals(numRegions * replication, tableRegions.size());
443 return table;
444 }
445
446 private static byte[][] makeN(byte[] base, int n) {
447 byte[][] ret = new byte[n][];
448 for (int i = 0; i < n; i++) {
449 ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%04d", i)));
450 }
451 return ret;
452 }
453
454 private void loadData(Table table) throws IOException {
455 for (int i = 0; i < ROWSIZE; i++) {
456 Put put = new Put(ROWS[i]);
457 put.add(FAMILYNAME, QUALIFIER, Bytes.toBytes(i));
458 table.put(put);
459 }
460 }
461
462 private void verifyRowCount(Table table, int expectedRegionNum)
463 throws IOException {
464 ResultScanner scanner = table.getScanner(new Scan());
465 int rowCount = 0;
466 while (scanner.next() != null) {
467 rowCount++;
468 }
469 assertEquals(expectedRegionNum, rowCount);
470 scanner.close();
471 }
472 }