1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import static org.apache.hadoop.hbase.util.HFileArchiveTestingUtil.assertArchiveEqualToOriginal;
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertFalse;
24 import static org.junit.Assert.assertTrue;
25 import static org.mockito.Mockito.doReturn;
26 import static org.mockito.Mockito.spy;
27
28 import java.io.IOException;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.SortedMap;
32 import java.util.TreeMap;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataOutputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.hbase.ChoreService;
42 import org.apache.hadoop.hbase.CoordinatedStateManager;
43 import org.apache.hadoop.hbase.HBaseTestingUtility;
44 import org.apache.hadoop.hbase.HColumnDescriptor;
45 import org.apache.hadoop.hbase.HConstants;
46 import org.apache.hadoop.hbase.HRegionInfo;
47 import org.apache.hadoop.hbase.HTableDescriptor;
48 import org.apache.hadoop.hbase.MetaMockingUtil;
49 import org.apache.hadoop.hbase.NamespaceDescriptor;
50 import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
51 import org.apache.hadoop.hbase.ProcedureInfo;
52 import org.apache.hadoop.hbase.Server;
53 import org.apache.hadoop.hbase.ServerName;
54 import org.apache.hadoop.hbase.testclassification.SmallTests;
55 import org.apache.hadoop.hbase.TableDescriptors;
56 import org.apache.hadoop.hbase.TableName;
57 import org.apache.hadoop.hbase.client.ClusterConnection;
58 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
59 import org.apache.hadoop.hbase.client.Result;
60 import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
61 import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination;
62 import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination.SplitLogManagerDetails;
63 import org.apache.hadoop.hbase.executor.ExecutorService;
64 import org.apache.hadoop.hbase.io.Reference;
65 import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
66 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
67 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
68 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
69 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
70 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
71 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest;
72 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiResponse;
73 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateRequest;
74 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateResponse;
75 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionAction;
76 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
77 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultOrException;
78 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
79 import org.apache.hadoop.hbase.regionserver.HStore;
80 import org.apache.hadoop.hbase.util.Bytes;
81 import org.apache.hadoop.hbase.util.FSUtils;
82 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
83 import org.apache.hadoop.hbase.util.Triple;
84 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
85 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
86 import org.junit.Test;
87 import org.junit.experimental.categories.Category;
88 import org.mockito.Mockito;
89 import org.mockito.invocation.InvocationOnMock;
90 import org.mockito.stubbing.Answer;
91
92 import com.google.protobuf.RpcController;
93 import com.google.protobuf.Service;
94 import com.google.protobuf.ServiceException;
95
96 @Category(SmallTests.class)
97 public class TestCatalogJanitor {
98 private static final Log LOG = LogFactory.getLog(TestCatalogJanitor.class);
99
100
101
102
103
104 class MockServer implements Server {
105 private final ClusterConnection connection;
106 private final Configuration c;
107
108 MockServer(final HBaseTestingUtility htu)
109 throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
110 this.c = htu.getConfiguration();
111 ClientProtos.ClientService.BlockingInterface ri =
112 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
113 MutateResponse.Builder builder = MutateResponse.newBuilder();
114 builder.setProcessed(true);
115 try {
116 Mockito.when(ri.mutate(
117 (RpcController)Mockito.any(), (MutateRequest)Mockito.any())).
118 thenReturn(builder.build());
119 } catch (ServiceException se) {
120 throw ProtobufUtil.getRemoteException(se);
121 }
122 try {
123 Mockito.when(ri.multi(
124 (RpcController)Mockito.any(), (MultiRequest)Mockito.any())).
125 thenAnswer(new Answer<MultiResponse>() {
126 @Override
127 public MultiResponse answer(InvocationOnMock invocation) throws Throwable {
128 return buildMultiResponse( (MultiRequest)invocation.getArguments()[1]);
129 }
130 });
131 } catch (ServiceException se) {
132 throw ProtobufUtil.getRemoteException(se);
133 }
134
135
136
137 this.connection =
138 HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
139 Mockito.mock(AdminProtos.AdminService.BlockingInterface.class), ri,
140 ServerName.valueOf("example.org,12345,6789"),
141 HRegionInfo.FIRST_META_REGIONINFO);
142
143 FileSystem fs = FileSystem.get(this.c);
144 Path rootdir = FSUtils.getRootDir(this.c);
145 FSUtils.setRootDir(this.c, rootdir);
146 AdminProtos.AdminService.BlockingInterface hri =
147 Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
148 }
149
150 @Override
151 public ClusterConnection getConnection() {
152 return this.connection;
153 }
154
155 @Override
156 public MetaTableLocator getMetaTableLocator() {
157 return null;
158 }
159
160 @Override
161 public Configuration getConfiguration() {
162 return this.c;
163 }
164
165 @Override
166 public ServerName getServerName() {
167 return ServerName.valueOf("mockserver.example.org", 1234, -1L);
168 }
169
170 @Override
171 public ZooKeeperWatcher getZooKeeper() {
172 return null;
173 }
174
175 @Override
176 public CoordinatedStateManager getCoordinatedStateManager() {
177 BaseCoordinatedStateManager m = Mockito.mock(BaseCoordinatedStateManager.class);
178 SplitLogManagerCoordination c = Mockito.mock(SplitLogManagerCoordination.class);
179 Mockito.when(m.getSplitLogManagerCoordination()).thenReturn(c);
180 SplitLogManagerDetails d = Mockito.mock(SplitLogManagerDetails.class);
181 Mockito.when(c.getDetails()).thenReturn(d);
182 return m;
183 }
184
185 @Override
186 public void abort(String why, Throwable e) {
187
188 }
189
190 @Override
191 public boolean isAborted() {
192 return false;
193 }
194
195 @Override
196 public boolean isStopped() {
197 return false;
198 }
199
200 @Override
201 public void stop(String why) {
202 }
203
204 @Override
205 public ChoreService getChoreService() {
206 return null;
207 }
208 }
209
210
211
212
213 class MockMasterServices implements MasterServices {
214 private final MasterFileSystem mfs;
215 private final AssignmentManager asm;
216
217 MockMasterServices(final Server server) throws IOException {
218 this.mfs = new MasterFileSystem(server, this);
219 this.asm = Mockito.mock(AssignmentManager.class);
220 }
221
222 @Override
223 public void checkTableModifiable(TableName tableName) throws IOException {
224
225 }
226
227 @Override
228 public boolean isMasterProcedureExecutorEnabled() {
229 return true;
230 }
231
232 @Override
233 public long createTable(
234 final HTableDescriptor desc,
235 final byte[][] splitKeys,
236 final long nonceGroup,
237 final long nonce) throws IOException {
238
239 return -1;
240 }
241
242 @Override
243 public AssignmentManager getAssignmentManager() {
244 return this.asm;
245 }
246
247 @Override
248 public ExecutorService getExecutorService() {
249 return null;
250 }
251
252 @Override
253 public ChoreService getChoreService() {
254 return null;
255 }
256
257 @Override
258 public MasterFileSystem getMasterFileSystem() {
259 return this.mfs;
260 }
261
262 @Override
263 public MasterCoprocessorHost getMasterCoprocessorHost() {
264 return null;
265 }
266
267 @Override
268 public MasterQuotaManager getMasterQuotaManager() {
269 return null;
270 }
271
272 @Override
273 public ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
274 return null;
275 }
276
277 @Override
278 public ServerManager getServerManager() {
279 return null;
280 }
281
282 @Override
283 public ZooKeeperWatcher getZooKeeper() {
284 return null;
285 }
286
287 @Override
288 public CoordinatedStateManager getCoordinatedStateManager() {
289 return null;
290 }
291
292 @Override
293 public MetaTableLocator getMetaTableLocator() {
294 return null;
295 }
296
297 @Override
298 public ClusterConnection getConnection() {
299 return null;
300 }
301
302 @Override
303 public Configuration getConfiguration() {
304 return mfs.conf;
305 }
306
307 @Override
308 public ServerName getServerName() {
309 return null;
310 }
311
312 @Override
313 public void abort(String why, Throwable e) {
314
315 }
316
317 @Override
318 public boolean isAborted() {
319 return false;
320 }
321
322 private boolean stopped = false;
323
324 @Override
325 public void stop(String why) {
326 stopped = true;
327 }
328
329 @Override
330 public boolean isStopped() {
331 return stopped;
332 }
333
334 @Override
335 public TableDescriptors getTableDescriptors() {
336 return new TableDescriptors() {
337 @Override
338 public HTableDescriptor remove(TableName tablename) throws IOException {
339
340 return null;
341 }
342
343 @Override
344 public Map<String, HTableDescriptor> getAll() throws IOException {
345
346 return null;
347 }
348
349 @Override
350 public HTableDescriptor get(TableName tablename)
351 throws IOException {
352 return createHTableDescriptor();
353 }
354
355 @Override
356 public Map<String, HTableDescriptor> getByNamespace(String name) throws IOException {
357 return null;
358 }
359
360 @Override
361 public void add(HTableDescriptor htd) throws IOException {
362
363
364 }
365 @Override
366 public void setCacheOn() throws IOException {
367 }
368
369 @Override
370 public void setCacheOff() throws IOException {
371 }
372 };
373 }
374
375 @Override
376 public boolean isServerShutdownHandlerEnabled() {
377 return true;
378 }
379
380 @Override
381 public boolean registerService(Service instance) {
382 return false;
383 }
384
385 @Override
386 public void createNamespace(NamespaceDescriptor descriptor) throws IOException {
387
388 }
389
390 @Override
391 public void modifyNamespace(NamespaceDescriptor descriptor) throws IOException {
392
393 }
394
395 @Override
396 public void deleteNamespace(String name) throws IOException {
397
398 }
399
400 @Override
401 public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
402 return null;
403 }
404
405 @Override
406 public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
407 return null;
408 }
409
410 @Override
411 public boolean abortProcedure(final long procId, final boolean mayInterruptIfRunning)
412 throws IOException {
413 return false;
414 }
415
416 @Override
417 public List<ProcedureInfo> listProcedures() throws IOException {
418 return null;
419 }
420
421 @Override
422 public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
423 return null;
424 }
425
426 @Override
427 public List<TableName> listTableNamesByNamespace(String name) throws IOException {
428 return null;
429 }
430
431 @Override
432 public long deleteTable(
433 final TableName tableName,
434 final long nonceGroup,
435 final long nonce) throws IOException {
436 return -1;
437 }
438
439 public void truncateTable(
440 final TableName tableName,
441 final boolean preserveSplits,
442 final long nonceGroup,
443 final long nonce) throws IOException {
444 }
445
446 @Override
447 public void modifyTable(
448 final TableName tableName,
449 final HTableDescriptor descriptor,
450 final long nonceGroup,
451 final long nonce) throws IOException {
452 }
453
454 @Override
455 public long enableTable(
456 final TableName tableName,
457 final long nonceGroup,
458 final long nonce) throws IOException {
459 return -1;
460 }
461
462 @Override
463 public long disableTable(
464 TableName tableName,
465 final long nonceGroup,
466 final long nonce) throws IOException {
467 return -1;
468 }
469
470 @Override
471 public void addColumn(
472 final TableName tableName,
473 final HColumnDescriptor columnDescriptor,
474 final long nonceGroup,
475 final long nonce) throws IOException { }
476
477 @Override
478 public void modifyColumn(
479 final TableName tableName,
480 final HColumnDescriptor descriptor,
481 final long nonceGroup,
482 final long nonce) throws IOException { }
483
484 @Override
485 public void deleteColumn(
486 final TableName tableName,
487 final byte[] columnName,
488 final long nonceGroup,
489 final long nonce) throws IOException { }
490
491 @Override
492 public TableLockManager getTableLockManager() {
493 return null;
494 }
495
496 @Override
497 public void dispatchMergingRegions(HRegionInfo region_a, HRegionInfo region_b,
498 boolean forcible) throws IOException {
499 }
500
501 @Override
502 public boolean isInitialized() {
503
504 return false;
505 }
506
507 @Override
508 public long getLastMajorCompactionTimestamp(TableName table) throws IOException {
509
510 return 0;
511 }
512
513 @Override
514 public long getLastMajorCompactionTimestampForRegion(byte[] regionName) throws IOException {
515
516 return 0;
517 }
518 }
519
520 @Test
521 public void testCleanParent() throws IOException, InterruptedException {
522 HBaseTestingUtility htu = new HBaseTestingUtility();
523 setRootDirAndCleanIt(htu, "testCleanParent");
524 Server server = new MockServer(htu);
525 try {
526 MasterServices services = new MockMasterServices(server);
527 CatalogJanitor janitor = new CatalogJanitor(server, services);
528
529 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
530 htd.addFamily(new HColumnDescriptor("f"));
531 HRegionInfo parent =
532 new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
533 Bytes.toBytes("eee"));
534 HRegionInfo splita =
535 new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
536 Bytes.toBytes("ccc"));
537 HRegionInfo splitb =
538 new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
539 Bytes.toBytes("eee"));
540
541
542 Result r = createResult(parent, splita, splitb);
543
544 Path rootdir = services.getMasterFileSystem().getRootDir();
545 Path tabledir =
546 FSUtils.getTableDir(rootdir, htd.getTableName());
547 Path storedir = HStore.getStoreHomedir(tabledir, splita,
548 htd.getColumnFamilies()[0].getName());
549 Reference ref = Reference.createTopReference(Bytes.toBytes("ccc"));
550 long now = System.currentTimeMillis();
551
552 Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
553 FileSystem fs = services.getMasterFileSystem().getFileSystem();
554 Path path = ref.write(fs, p);
555 assertTrue(fs.exists(path));
556 assertFalse(janitor.cleanParent(parent, r));
557
558 assertTrue(fs.delete(p, true));
559 assertTrue(janitor.cleanParent(parent, r));
560 } finally {
561 server.stop("shutdown");
562 }
563 }
564
565
566
567
568
569
570 @Test
571 public void testParentCleanedEvenIfDaughterGoneFirst()
572 throws IOException, InterruptedException {
573 parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
574 "testParentCleanedEvenIfDaughterGoneFirst", Bytes.toBytes("eee"));
575 }
576
577
578
579
580
581
582 @Test
583 public void testLastParentCleanedEvenIfDaughterGoneFirst()
584 throws IOException, InterruptedException {
585 parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
586 "testLastParentCleanedEvenIfDaughterGoneFirst", new byte[0]);
587 }
588
589
590
591
592
593
594
595
596
597 private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
598 final String rootDir, final byte[] lastEndKey)
599 throws IOException, InterruptedException {
600 HBaseTestingUtility htu = new HBaseTestingUtility();
601 setRootDirAndCleanIt(htu, rootDir);
602 Server server = new MockServer(htu);
603 MasterServices services = new MockMasterServices(server);
604 CatalogJanitor janitor = new CatalogJanitor(server, services);
605 final HTableDescriptor htd = createHTableDescriptor();
606
607
608
609
610 HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
611 lastEndKey);
612
613
614 Thread.sleep(1001);
615
616
617 HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
618 Bytes.toBytes("ccc"));
619 Thread.sleep(1001);
620
621 HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
622 Bytes.toBytes("bbb"));
623 HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
624 Bytes.toBytes("ccc"));
625
626
627 HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
628 lastEndKey);
629 Thread.sleep(1001);
630
631 HRegionInfo splitba = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
632 Bytes.toBytes("ddd"));
633 HRegionInfo splitbb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ddd"),
634 lastEndKey);
635
636
637
638 SortedMap<HRegionInfo, Result> regions =
639 new TreeMap<HRegionInfo, Result>(new CatalogJanitor.SplitParentFirstComparator());
640
641 regions.put(parent, createResult(parent, splita, splitb));
642 regions.put(splitb, createResult(splitb, splitba, splitbb));
643 regions.put(splita, createResult(splita, splitaa, splitab));
644
645 int index = 0;
646 for (Map.Entry<HRegionInfo, Result> e: regions.entrySet()) {
647 if (index == 0) {
648 assertTrue(e.getKey().getEncodedName().equals(parent.getEncodedName()));
649 } else if (index == 1) {
650 assertTrue(e.getKey().getEncodedName().equals(splita.getEncodedName()));
651 } else if (index == 2) {
652 assertTrue(e.getKey().getEncodedName().equals(splitb.getEncodedName()));
653 }
654 index++;
655 }
656
657
658
659 Path splitaRef =
660 createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
661
662 assertFalse(janitor.cleanParent(parent, regions.get(parent)));
663
664
665
666 assertTrue(janitor.cleanParent(splitb, regions.get(splitb)));
667
668
669
670
671
672 FileSystem fs = FileSystem.get(htu.getConfiguration());
673 assertTrue(fs.delete(splitaRef, true));
674
675 Path splitaaRef =
676 createReferences(services, htd, splita, splitaa, Bytes.toBytes("bbb"), false);
677 Path splitabRef =
678 createReferences(services, htd, splita, splitab, Bytes.toBytes("bbb"), true);
679
680
681 assertFalse(janitor.cleanParent(splita, regions.get(splita)));
682
683
684 assertTrue(fs.delete(splitaaRef, true));
685 assertTrue(fs.delete(splitabRef, true));
686 assertTrue(janitor.cleanParent(splita, regions.get(splita)));
687
688
689 assertTrue(janitor.cleanParent(parent, regions.get(parent)));
690
691 services.stop("test finished");
692 janitor.cancel(true);
693 }
694
695
696
697
698
699
700 @Test
701 public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception {
702 HBaseTestingUtility htu = new HBaseTestingUtility();
703 setRootDirAndCleanIt(htu, "testScanDoesNotCleanRegionsWithExistingParents");
704 Server server = new MockServer(htu);
705 MasterServices services = new MockMasterServices(server);
706
707 final HTableDescriptor htd = createHTableDescriptor();
708
709
710
711
712 HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
713 new byte[0], true);
714
715
716 Thread.sleep(1001);
717
718
719 HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
720 Bytes.toBytes("ccc"), true);
721 Thread.sleep(1001);
722
723 HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
724 Bytes.toBytes("bbb"), false);
725 HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
726 Bytes.toBytes("ccc"), false);
727
728
729 HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
730 new byte[0]);
731 Thread.sleep(1001);
732
733 final Map<HRegionInfo, Result> splitParents =
734 new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
735 splitParents.put(parent, createResult(parent, splita, splitb));
736 splita.setOffline(true);
737 splitParents.put(splita, createResult(splita, splitaa,splitab));
738
739 final Map<HRegionInfo, Result> mergedRegions = new TreeMap<HRegionInfo, Result>();
740 CatalogJanitor janitor = spy(new CatalogJanitor(server, services));
741 doReturn(new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>(
742 10, mergedRegions, splitParents)).when(janitor)
743 .getMergedRegionsAndSplitParents();
744
745
746 Path splitaRef =
747 createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
748
749
750 assertEquals(0, janitor.scan());
751
752
753 FileSystem fs = FileSystem.get(htu.getConfiguration());
754 assertTrue(fs.delete(splitaRef, true));
755
756
757 assertEquals(2, janitor.scan());
758
759 services.stop("test finished");
760 janitor.cancel(true);
761 }
762
763
764
765
766
767 @Test
768 public void testSplitParentFirstComparator() {
769 SplitParentFirstComparator comp = new SplitParentFirstComparator();
770 final HTableDescriptor htd = createHTableDescriptor();
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791 HRegionInfo rootRegion = new HRegionInfo(htd.getTableName(),
792 HConstants.EMPTY_START_ROW,
793 HConstants.EMPTY_END_ROW, true);
794 HRegionInfo firstRegion = new HRegionInfo(htd.getTableName(),
795 HConstants.EMPTY_START_ROW,
796 Bytes.toBytes("bbb"), true);
797 HRegionInfo lastRegion = new HRegionInfo(htd.getTableName(),
798 Bytes.toBytes("bbb"),
799 HConstants.EMPTY_END_ROW, true);
800
801 assertTrue(comp.compare(rootRegion, rootRegion) == 0);
802 assertTrue(comp.compare(firstRegion, firstRegion) == 0);
803 assertTrue(comp.compare(lastRegion, lastRegion) == 0);
804 assertTrue(comp.compare(rootRegion, firstRegion) < 0);
805 assertTrue(comp.compare(rootRegion, lastRegion) < 0);
806 assertTrue(comp.compare(firstRegion, lastRegion) < 0);
807
808
809 HRegionInfo firstRegiona = new HRegionInfo(htd.getTableName(),
810 HConstants.EMPTY_START_ROW,
811 Bytes.toBytes("aaa"), true);
812 HRegionInfo firstRegionb = new HRegionInfo(htd.getTableName(),
813 Bytes.toBytes("aaa"),
814 Bytes.toBytes("bbb"), true);
815
816 HRegionInfo lastRegiona = new HRegionInfo(htd.getTableName(),
817 Bytes.toBytes("bbb"),
818 Bytes.toBytes("ddd"), true);
819 HRegionInfo lastRegionb = new HRegionInfo(htd.getTableName(),
820 Bytes.toBytes("ddd"),
821 HConstants.EMPTY_END_ROW, true);
822
823 assertTrue(comp.compare(firstRegiona, firstRegiona) == 0);
824 assertTrue(comp.compare(firstRegionb, firstRegionb) == 0);
825 assertTrue(comp.compare(rootRegion, firstRegiona) < 0);
826 assertTrue(comp.compare(rootRegion, firstRegionb) < 0);
827 assertTrue(comp.compare(firstRegion, firstRegiona) < 0);
828 assertTrue(comp.compare(firstRegion, firstRegionb) < 0);
829 assertTrue(comp.compare(firstRegiona, firstRegionb) < 0);
830
831 assertTrue(comp.compare(lastRegiona, lastRegiona) == 0);
832 assertTrue(comp.compare(lastRegionb, lastRegionb) == 0);
833 assertTrue(comp.compare(rootRegion, lastRegiona) < 0);
834 assertTrue(comp.compare(rootRegion, lastRegionb) < 0);
835 assertTrue(comp.compare(lastRegion, lastRegiona) < 0);
836 assertTrue(comp.compare(lastRegion, lastRegionb) < 0);
837 assertTrue(comp.compare(lastRegiona, lastRegionb) < 0);
838
839 assertTrue(comp.compare(firstRegiona, lastRegiona) < 0);
840 assertTrue(comp.compare(firstRegiona, lastRegionb) < 0);
841 assertTrue(comp.compare(firstRegionb, lastRegiona) < 0);
842 assertTrue(comp.compare(firstRegionb, lastRegionb) < 0);
843
844 HRegionInfo lastRegionaa = new HRegionInfo(htd.getTableName(),
845 Bytes.toBytes("bbb"),
846 Bytes.toBytes("ccc"), false);
847 HRegionInfo lastRegionab = new HRegionInfo(htd.getTableName(),
848 Bytes.toBytes("ccc"),
849 Bytes.toBytes("ddd"), false);
850
851 assertTrue(comp.compare(lastRegiona, lastRegionaa) < 0);
852 assertTrue(comp.compare(lastRegiona, lastRegionab) < 0);
853 assertTrue(comp.compare(lastRegionaa, lastRegionab) < 0);
854
855 }
856
857 @Test
858 public void testArchiveOldRegion() throws Exception {
859 String table = "table";
860 HBaseTestingUtility htu = new HBaseTestingUtility();
861 setRootDirAndCleanIt(htu, "testCleanParent");
862 Server server = new MockServer(htu);
863 MasterServices services = new MockMasterServices(server);
864
865
866 CatalogJanitor janitor = new CatalogJanitor(server, services);
867
868
869 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
870 htd.addFamily(new HColumnDescriptor("f"));
871 HRegionInfo parent = new HRegionInfo(htd.getTableName(),
872 Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
873 HRegionInfo splita = new HRegionInfo(htd.getTableName(),
874 Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
875 HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
876 Bytes.toBytes("ccc"),
877 Bytes.toBytes("eee"));
878
879
880
881 Result parentMetaRow = createResult(parent, splita, splitb);
882 FileSystem fs = FileSystem.get(htu.getConfiguration());
883 Path rootdir = services.getMasterFileSystem().getRootDir();
884
885
886
887 FSUtils.setRootDir(fs.getConf(), rootdir);
888 Path tabledir = FSUtils.getTableDir(rootdir, htd.getTableName());
889 Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
890 Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
891 tabledir, htd.getColumnFamilies()[0].getName());
892 LOG.debug("Table dir:" + tabledir);
893 LOG.debug("Store dir:" + storedir);
894 LOG.debug("Store archive dir:" + storeArchive);
895
896
897 FileStatus[] mockFiles = addMockStoreFiles(2, services, storedir);
898
899 FileStatus[] storeFiles = fs.listStatus(storedir);
900 int index = 0;
901 for (FileStatus file : storeFiles) {
902 LOG.debug("Have store file:" + file.getPath());
903 assertEquals("Got unexpected store file", mockFiles[index].getPath(),
904 storeFiles[index].getPath());
905 index++;
906 }
907
908
909 assertTrue(janitor.cleanParent(parent, parentMetaRow));
910 LOG.debug("Finished cleanup of parent region");
911
912
913 FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
914 logFiles("archived files", storeFiles);
915 logFiles("archived files", archivedStoreFiles);
916
917 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
918
919
920 FSUtils.delete(fs, rootdir, true);
921 services.stop("Test finished");
922 server.stop("Test finished");
923 janitor.cancel(true);
924 }
925
926
927
928
929
930 private void logFiles(String description, FileStatus[] storeFiles) {
931 LOG.debug("Current " + description + ": ");
932 for (FileStatus file : storeFiles) {
933 LOG.debug(file.getPath());
934 }
935 }
936
937
938
939
940
941 @Test
942 public void testDuplicateHFileResolution() throws Exception {
943 String table = "table";
944 HBaseTestingUtility htu = new HBaseTestingUtility();
945 setRootDirAndCleanIt(htu, "testCleanParent");
946 Server server = new MockServer(htu);
947 MasterServices services = new MockMasterServices(server);
948
949
950
951 CatalogJanitor janitor = new CatalogJanitor(server, services);
952
953
954 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
955 htd.addFamily(new HColumnDescriptor("f"));
956 HRegionInfo parent = new HRegionInfo(htd.getTableName(),
957 Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
958 HRegionInfo splita = new HRegionInfo(htd.getTableName(),
959 Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
960 HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
961 Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
962
963
964 Result r = createResult(parent, splita, splitb);
965
966 FileSystem fs = FileSystem.get(htu.getConfiguration());
967
968 Path rootdir = services.getMasterFileSystem().getRootDir();
969
970
971
972 FSUtils.setRootDir(fs.getConf(), rootdir);
973 Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
974 Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
975 System.out.println("Old root:" + rootdir);
976 System.out.println("Old table:" + tabledir);
977 System.out.println("Old store:" + storedir);
978
979 Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
980 tabledir, htd.getColumnFamilies()[0].getName());
981 System.out.println("Old archive:" + storeArchive);
982
983
984 addMockStoreFiles(2, services, storedir);
985
986 FileStatus[] storeFiles = fs.listStatus(storedir);
987
988 assertTrue(janitor.cleanParent(parent, r));
989
990
991 FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
992 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
993
994
995
996 addMockStoreFiles(2, services, storedir);
997
998
999 assertTrue(janitor.cleanParent(parent, r));
1000
1001
1002 archivedStoreFiles = fs.listStatus(storeArchive);
1003 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs, true);
1004
1005
1006 services.stop("Test finished");
1007 server.stop("shutdown");
1008 janitor.cancel(true);
1009 }
1010
1011 private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir)
1012 throws IOException {
1013
1014 FileSystem fs = services.getMasterFileSystem().getFileSystem();
1015 fs.mkdirs(storedir);
1016
1017 for (int i = 0; i < count; i++) {
1018 Path storeFile = new Path(storedir, "_store" + i);
1019 FSDataOutputStream dos = fs.create(storeFile, true);
1020 dos.writeBytes("Some data: " + i);
1021 dos.close();
1022 }
1023 LOG.debug("Adding " + count + " store files to the storedir:" + storedir);
1024
1025 FileStatus[] storeFiles = fs.listStatus(storedir);
1026 assertEquals("Didn't have expected store files", count, storeFiles.length);
1027 return storeFiles;
1028 }
1029
1030 private String setRootDirAndCleanIt(final HBaseTestingUtility htu,
1031 final String subdir)
1032 throws IOException {
1033 Path testdir = htu.getDataTestDir(subdir);
1034 FileSystem fs = FileSystem.get(htu.getConfiguration());
1035 if (fs.exists(testdir)) assertTrue(fs.delete(testdir, true));
1036 FSUtils.setRootDir(htu.getConfiguration(), testdir);
1037 return FSUtils.getRootDir(htu.getConfiguration()).toString();
1038 }
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050 private Path createReferences(final MasterServices services,
1051 final HTableDescriptor htd, final HRegionInfo parent,
1052 final HRegionInfo daughter, final byte [] midkey, final boolean top)
1053 throws IOException {
1054 Path rootdir = services.getMasterFileSystem().getRootDir();
1055 Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
1056 Path storedir = HStore.getStoreHomedir(tabledir, daughter,
1057 htd.getColumnFamilies()[0].getName());
1058 Reference ref =
1059 top? Reference.createTopReference(midkey): Reference.createBottomReference(midkey);
1060 long now = System.currentTimeMillis();
1061
1062 Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
1063 FileSystem fs = services.getMasterFileSystem().getFileSystem();
1064 ref.write(fs, p);
1065 return p;
1066 }
1067
1068 private Result createResult(final HRegionInfo parent, final HRegionInfo a,
1069 final HRegionInfo b)
1070 throws IOException {
1071 return MetaMockingUtil.getMetaTableRowResult(parent, null, a, b);
1072 }
1073
1074 private HTableDescriptor createHTableDescriptor() {
1075 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("t"));
1076 htd.addFamily(new HColumnDescriptor("f"));
1077 return htd;
1078 }
1079
1080 private MultiResponse buildMultiResponse(MultiRequest req) {
1081 MultiResponse.Builder builder = MultiResponse.newBuilder();
1082 RegionActionResult.Builder regionActionResultBuilder =
1083 RegionActionResult.newBuilder();
1084 ResultOrException.Builder roeBuilder = ResultOrException.newBuilder();
1085 for (RegionAction regionAction: req.getRegionActionList()) {
1086 regionActionResultBuilder.clear();
1087 for (ClientProtos.Action action: regionAction.getActionList()) {
1088 roeBuilder.clear();
1089 roeBuilder.setResult(ClientProtos.Result.getDefaultInstance());
1090 roeBuilder.setIndex(action.getIndex());
1091 regionActionResultBuilder.addResultOrException(roeBuilder.build());
1092 }
1093 builder.addRegionActionResult(regionActionResultBuilder.build());
1094 }
1095 return builder.build();
1096 }
1097
1098 }
1099