1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import static org.apache.hadoop.hbase.util.HFileArchiveTestingUtil.assertArchiveEqualToOriginal;
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertFalse;
24 import static org.junit.Assert.assertTrue;
25 import static org.mockito.Mockito.doReturn;
26 import static org.mockito.Mockito.spy;
27
28 import java.io.IOException;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.SortedMap;
32 import java.util.TreeMap;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataOutputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.hbase.ChoreService;
42 import org.apache.hadoop.hbase.CoordinatedStateManager;
43 import org.apache.hadoop.hbase.HBaseTestingUtility;
44 import org.apache.hadoop.hbase.HColumnDescriptor;
45 import org.apache.hadoop.hbase.HConstants;
46 import org.apache.hadoop.hbase.HRegionInfo;
47 import org.apache.hadoop.hbase.HTableDescriptor;
48 import org.apache.hadoop.hbase.MetaMockingUtil;
49 import org.apache.hadoop.hbase.NamespaceDescriptor;
50 import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
51 import org.apache.hadoop.hbase.ProcedureInfo;
52 import org.apache.hadoop.hbase.Server;
53 import org.apache.hadoop.hbase.ServerName;
54 import org.apache.hadoop.hbase.testclassification.SmallTests;
55 import org.apache.hadoop.hbase.TableDescriptors;
56 import org.apache.hadoop.hbase.TableName;
57 import org.apache.hadoop.hbase.client.ClusterConnection;
58 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
59 import org.apache.hadoop.hbase.client.Result;
60 import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
61 import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination;
62 import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination.SplitLogManagerDetails;
63 import org.apache.hadoop.hbase.executor.ExecutorService;
64 import org.apache.hadoop.hbase.io.Reference;
65 import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
66 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
67 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
68 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
69 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
70 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
71 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
72 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest;
73 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiResponse;
74 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateRequest;
75 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateResponse;
76 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionAction;
77 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
78 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultOrException;
79 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
80 import org.apache.hadoop.hbase.regionserver.HStore;
81 import org.apache.hadoop.hbase.security.User;
82 import org.apache.hadoop.hbase.util.Bytes;
83 import org.apache.hadoop.hbase.util.FSUtils;
84 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
85 import org.apache.hadoop.hbase.util.Triple;
86 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
87 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
88 import org.junit.Test;
89 import org.junit.experimental.categories.Category;
90 import org.mockito.Mockito;
91 import org.mockito.invocation.InvocationOnMock;
92 import org.mockito.stubbing.Answer;
93
94 import com.google.protobuf.RpcController;
95 import com.google.protobuf.Service;
96 import com.google.protobuf.ServiceException;
97
98 @Category(SmallTests.class)
99 public class TestCatalogJanitor {
100 private static final Log LOG = LogFactory.getLog(TestCatalogJanitor.class);
101
102
103
104
105
106 class MockServer implements Server {
107 private final ClusterConnection connection;
108 private final Configuration c;
109
110 MockServer(final HBaseTestingUtility htu)
111 throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
112 this.c = htu.getConfiguration();
113 ClientProtos.ClientService.BlockingInterface ri =
114 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
115 MutateResponse.Builder builder = MutateResponse.newBuilder();
116 builder.setProcessed(true);
117 try {
118 Mockito.when(ri.mutate(
119 (RpcController)Mockito.any(), (MutateRequest)Mockito.any())).
120 thenReturn(builder.build());
121 } catch (ServiceException se) {
122 throw ProtobufUtil.getRemoteException(se);
123 }
124 try {
125 Mockito.when(ri.multi(
126 (RpcController)Mockito.any(), (MultiRequest)Mockito.any())).
127 thenAnswer(new Answer<MultiResponse>() {
128 @Override
129 public MultiResponse answer(InvocationOnMock invocation) throws Throwable {
130 return buildMultiResponse( (MultiRequest)invocation.getArguments()[1]);
131 }
132 });
133 } catch (ServiceException se) {
134 throw ProtobufUtil.getRemoteException(se);
135 }
136
137
138
139 this.connection =
140 HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
141 Mockito.mock(AdminProtos.AdminService.BlockingInterface.class), ri,
142 ServerName.valueOf("example.org,12345,6789"),
143 HRegionInfo.FIRST_META_REGIONINFO);
144
145 FileSystem fs = FileSystem.get(this.c);
146 Path rootdir = FSUtils.getRootDir(this.c);
147 FSUtils.setRootDir(this.c, rootdir);
148 AdminProtos.AdminService.BlockingInterface hri =
149 Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
150 }
151
152 @Override
153 public ClusterConnection getConnection() {
154 return this.connection;
155 }
156
157 @Override
158 public MetaTableLocator getMetaTableLocator() {
159 return null;
160 }
161
162 @Override
163 public Configuration getConfiguration() {
164 return this.c;
165 }
166
167 @Override
168 public ServerName getServerName() {
169 return ServerName.valueOf("mockserver.example.org", 1234, -1L);
170 }
171
172 @Override
173 public ZooKeeperWatcher getZooKeeper() {
174 return null;
175 }
176
177 @Override
178 public CoordinatedStateManager getCoordinatedStateManager() {
179 BaseCoordinatedStateManager m = Mockito.mock(BaseCoordinatedStateManager.class);
180 SplitLogManagerCoordination c = Mockito.mock(SplitLogManagerCoordination.class);
181 Mockito.when(m.getSplitLogManagerCoordination()).thenReturn(c);
182 SplitLogManagerDetails d = Mockito.mock(SplitLogManagerDetails.class);
183 Mockito.when(c.getDetails()).thenReturn(d);
184 return m;
185 }
186
187 @Override
188 public void abort(String why, Throwable e) {
189
190 }
191
192 @Override
193 public boolean isAborted() {
194 return false;
195 }
196
197 @Override
198 public boolean isStopped() {
199 return false;
200 }
201
202 @Override
203 public void stop(String why) {
204 }
205
206 @Override
207 public ChoreService getChoreService() {
208 return null;
209 }
210 }
211
212
213
214
215 class MockMasterServices implements MasterServices {
216 private final MasterFileSystem mfs;
217 private final AssignmentManager asm;
218 private final ServerManager sm;
219
220 MockMasterServices(final Server server) throws IOException {
221 this.mfs = new MasterFileSystem(server, this);
222 this.asm = Mockito.mock(AssignmentManager.class);
223 this.sm = Mockito.mock(ServerManager.class);
224 }
225
226 @Override
227 public void checkTableModifiable(TableName tableName) throws IOException {
228
229 }
230
231 @Override
232 public long createTable(
233 final HTableDescriptor desc,
234 final byte[][] splitKeys,
235 final long nonceGroup,
236 final long nonce) throws IOException {
237
238 return -1;
239 }
240
241 @Override
242 public long createSystemTable(final HTableDescriptor hTableDescriptor) throws IOException {
243 return -1;
244 }
245
246 @Override
247 public SnapshotManager getSnapshotManager() {
248 return null;
249 }
250
251 @Override
252 public AssignmentManager getAssignmentManager() {
253 return this.asm;
254 }
255
256 @Override
257 public ExecutorService getExecutorService() {
258 return null;
259 }
260
261 @Override
262 public ChoreService getChoreService() {
263 return null;
264 }
265
266 @Override
267 public MasterFileSystem getMasterFileSystem() {
268 return this.mfs;
269 }
270
271 @Override
272 public MasterCoprocessorHost getMasterCoprocessorHost() {
273 return null;
274 }
275
276 @Override
277 public MasterQuotaManager getMasterQuotaManager() {
278 return null;
279 }
280
281 @Override
282 public ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
283 return null;
284 }
285
286 @Override
287 public LoadBalancer getLoadBalancer() {
288 return null;
289 }
290
291 @Override
292 public ServerManager getServerManager() {
293 return sm;
294 }
295
296 @Override
297 public ZooKeeperWatcher getZooKeeper() {
298 return null;
299 }
300
301 @Override
302 public CoordinatedStateManager getCoordinatedStateManager() {
303 return null;
304 }
305
306 @Override
307 public MetaTableLocator getMetaTableLocator() {
308 return null;
309 }
310
311 @Override
312 public ClusterConnection getConnection() {
313 return null;
314 }
315
316 @Override
317 public Configuration getConfiguration() {
318 return mfs.conf;
319 }
320
321 @Override
322 public ServerName getServerName() {
323 return null;
324 }
325
326 @Override
327 public void abort(String why, Throwable e) {
328
329 }
330
331 @Override
332 public boolean isAborted() {
333 return false;
334 }
335
336 private boolean stopped = false;
337
338 @Override
339 public void stop(String why) {
340 stopped = true;
341 }
342
343 @Override
344 public boolean isStopped() {
345 return stopped;
346 }
347
348 @Override
349 public TableDescriptors getTableDescriptors() {
350 return new TableDescriptors() {
351 @Override
352 public HTableDescriptor remove(TableName tablename) throws IOException {
353
354 return null;
355 }
356
357 @Override
358 public Map<String, HTableDescriptor> getAll() throws IOException {
359
360 return null;
361 }
362
363 @Override
364 public HTableDescriptor get(TableName tablename)
365 throws IOException {
366 return createHTableDescriptor();
367 }
368
369 @Override
370 public Map<String, HTableDescriptor> getByNamespace(String name) throws IOException {
371 return null;
372 }
373
374 @Override
375 public void add(HTableDescriptor htd) throws IOException {
376
377
378 }
379 @Override
380 public void setCacheOn() throws IOException {
381 }
382
383 @Override
384 public void setCacheOff() throws IOException {
385 }
386 };
387 }
388
389 @Override
390 public boolean isServerCrashProcessingEnabled() {
391 return true;
392 }
393
394 @Override
395 public boolean registerService(Service instance) {
396 return false;
397 }
398
399 @Override
400 public void createNamespace(
401 final NamespaceDescriptor descriptor,
402 final long nonceGroup,
403 final long nonce) throws IOException {
404
405 }
406
407 @Override
408 public void createNamespaceSync(
409 final NamespaceDescriptor descriptor,
410 final long nonceGroup,
411 final long nonce,
412 final boolean executeCoprocessor) throws IOException {
413
414 }
415
416 @Override
417 public void modifyNamespace(
418 final NamespaceDescriptor descriptor,
419 final long nonceGroup,
420 final long nonce) throws IOException {
421
422 }
423
424 @Override
425 public void deleteNamespace(
426 final String name,
427 final long nonceGroup,
428 final long nonce) throws IOException {
429
430 }
431
432 @Override
433 public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
434 return null;
435 }
436
437 @Override
438 public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
439 return null;
440 }
441
442 @Override
443 public boolean abortProcedure(final long procId, final boolean mayInterruptIfRunning)
444 throws IOException {
445 return false;
446 }
447
448 @Override
449 public List<ProcedureInfo> listProcedures() throws IOException {
450 return null;
451 }
452
453 @Override
454 public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
455 return null;
456 }
457
458 @Override
459 public List<TableName> listTableNamesByNamespace(String name) throws IOException {
460 return null;
461 }
462
463 @Override
464 public long deleteTable(
465 final TableName tableName,
466 final long nonceGroup,
467 final long nonce) throws IOException {
468 return -1;
469 }
470
471 public void truncateTable(
472 final TableName tableName,
473 final boolean preserveSplits,
474 final long nonceGroup,
475 final long nonce) throws IOException {
476 }
477
478 @Override
479 public void modifyTable(
480 final TableName tableName,
481 final HTableDescriptor descriptor,
482 final long nonceGroup,
483 final long nonce) throws IOException {
484 }
485
486 @Override
487 public long enableTable(
488 final TableName tableName,
489 final long nonceGroup,
490 final long nonce) throws IOException {
491 return -1;
492 }
493
494 @Override
495 public long disableTable(
496 TableName tableName,
497 final long nonceGroup,
498 final long nonce) throws IOException {
499 return -1;
500 }
501
502 @Override
503 public void addColumn(
504 final TableName tableName,
505 final HColumnDescriptor columnDescriptor,
506 final long nonceGroup,
507 final long nonce) throws IOException { }
508
509 @Override
510 public void modifyColumn(
511 final TableName tableName,
512 final HColumnDescriptor descriptor,
513 final long nonceGroup,
514 final long nonce) throws IOException { }
515
516 @Override
517 public void deleteColumn(
518 final TableName tableName,
519 final byte[] columnName,
520 final long nonceGroup,
521 final long nonce) throws IOException { }
522
523 @Override
524 public TableLockManager getTableLockManager() {
525 return null;
526 }
527
528 @Override
529 public TableNamespaceManager getTableNamespaceManager() {
530 return null;
531 }
532
533 @Override
534 public void dispatchMergingRegions(HRegionInfo region_a, HRegionInfo region_b,
535 boolean forcible, User user) throws IOException {
536 }
537
538 @Override
539 public boolean isInitialized() {
540
541 return false;
542 }
543
544 @Override
545 public long getLastMajorCompactionTimestamp(TableName table) throws IOException {
546
547 return 0;
548 }
549
550 @Override
551 public long getLastMajorCompactionTimestampForRegion(byte[] regionName) throws IOException {
552
553 return 0;
554 }
555 }
556
557 @Test
558 public void testCleanParent() throws IOException, InterruptedException {
559 HBaseTestingUtility htu = new HBaseTestingUtility();
560 setRootDirAndCleanIt(htu, "testCleanParent");
561 Server server = new MockServer(htu);
562 try {
563 MasterServices services = new MockMasterServices(server);
564 CatalogJanitor janitor = new CatalogJanitor(server, services);
565
566 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
567 htd.addFamily(new HColumnDescriptor("f"));
568 HRegionInfo parent =
569 new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
570 Bytes.toBytes("eee"));
571 HRegionInfo splita =
572 new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
573 Bytes.toBytes("ccc"));
574 HRegionInfo splitb =
575 new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
576 Bytes.toBytes("eee"));
577
578
579 Result r = createResult(parent, splita, splitb);
580
581 Path rootdir = services.getMasterFileSystem().getRootDir();
582 Path tabledir =
583 FSUtils.getTableDir(rootdir, htd.getTableName());
584 Path storedir = HStore.getStoreHomedir(tabledir, splita,
585 htd.getColumnFamilies()[0].getName());
586 Reference ref = Reference.createTopReference(Bytes.toBytes("ccc"));
587 long now = System.currentTimeMillis();
588
589 Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
590 FileSystem fs = services.getMasterFileSystem().getFileSystem();
591 Path path = ref.write(fs, p);
592 assertTrue(fs.exists(path));
593 assertFalse(janitor.cleanParent(parent, r));
594
595 assertTrue(fs.delete(p, true));
596 assertTrue(janitor.cleanParent(parent, r));
597 } finally {
598 server.stop("shutdown");
599 }
600 }
601
602
603
604
605
606
607 @Test
608 public void testParentCleanedEvenIfDaughterGoneFirst()
609 throws IOException, InterruptedException {
610 parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
611 "testParentCleanedEvenIfDaughterGoneFirst", Bytes.toBytes("eee"));
612 }
613
614
615
616
617
618
619 @Test
620 public void testLastParentCleanedEvenIfDaughterGoneFirst()
621 throws IOException, InterruptedException {
622 parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
623 "testLastParentCleanedEvenIfDaughterGoneFirst", new byte[0]);
624 }
625
626
627
628
629
630
631
632
633
634 private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
635 final String rootDir, final byte[] lastEndKey)
636 throws IOException, InterruptedException {
637 HBaseTestingUtility htu = new HBaseTestingUtility();
638 setRootDirAndCleanIt(htu, rootDir);
639 Server server = new MockServer(htu);
640 MasterServices services = new MockMasterServices(server);
641 CatalogJanitor janitor = new CatalogJanitor(server, services);
642 final HTableDescriptor htd = createHTableDescriptor();
643
644
645
646
647 HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
648 lastEndKey);
649
650
651 Thread.sleep(1001);
652
653
654 HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
655 Bytes.toBytes("ccc"));
656 Thread.sleep(1001);
657
658 HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
659 Bytes.toBytes("bbb"));
660 HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
661 Bytes.toBytes("ccc"));
662
663
664 HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
665 lastEndKey);
666 Thread.sleep(1001);
667
668 HRegionInfo splitba = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
669 Bytes.toBytes("ddd"));
670 HRegionInfo splitbb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ddd"),
671 lastEndKey);
672
673
674
675 SortedMap<HRegionInfo, Result> regions =
676 new TreeMap<HRegionInfo, Result>(new CatalogJanitor.SplitParentFirstComparator());
677
678 regions.put(parent, createResult(parent, splita, splitb));
679 regions.put(splitb, createResult(splitb, splitba, splitbb));
680 regions.put(splita, createResult(splita, splitaa, splitab));
681
682 int index = 0;
683 for (Map.Entry<HRegionInfo, Result> e: regions.entrySet()) {
684 if (index == 0) {
685 assertTrue(e.getKey().getEncodedName().equals(parent.getEncodedName()));
686 } else if (index == 1) {
687 assertTrue(e.getKey().getEncodedName().equals(splita.getEncodedName()));
688 } else if (index == 2) {
689 assertTrue(e.getKey().getEncodedName().equals(splitb.getEncodedName()));
690 }
691 index++;
692 }
693
694
695
696 Path splitaRef =
697 createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
698
699 assertFalse(janitor.cleanParent(parent, regions.get(parent)));
700
701
702
703 assertTrue(janitor.cleanParent(splitb, regions.get(splitb)));
704
705
706
707
708
709 FileSystem fs = FileSystem.get(htu.getConfiguration());
710 assertTrue(fs.delete(splitaRef, true));
711
712 Path splitaaRef =
713 createReferences(services, htd, splita, splitaa, Bytes.toBytes("bbb"), false);
714 Path splitabRef =
715 createReferences(services, htd, splita, splitab, Bytes.toBytes("bbb"), true);
716
717
718 assertFalse(janitor.cleanParent(splita, regions.get(splita)));
719
720
721 assertTrue(fs.delete(splitaaRef, true));
722 assertTrue(fs.delete(splitabRef, true));
723 assertTrue(janitor.cleanParent(splita, regions.get(splita)));
724
725
726 assertTrue(janitor.cleanParent(parent, regions.get(parent)));
727
728 services.stop("test finished");
729 janitor.cancel(true);
730 }
731
732
733
734
735
736
737 @Test
738 public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception {
739 HBaseTestingUtility htu = new HBaseTestingUtility();
740 setRootDirAndCleanIt(htu, "testScanDoesNotCleanRegionsWithExistingParents");
741 Server server = new MockServer(htu);
742 MasterServices services = new MockMasterServices(server);
743
744 final HTableDescriptor htd = createHTableDescriptor();
745
746
747
748
749 HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
750 new byte[0], true);
751
752
753 Thread.sleep(1001);
754
755
756 HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
757 Bytes.toBytes("ccc"), true);
758 Thread.sleep(1001);
759
760 HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
761 Bytes.toBytes("bbb"), false);
762 HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
763 Bytes.toBytes("ccc"), false);
764
765
766 HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
767 new byte[0]);
768 Thread.sleep(1001);
769
770 final Map<HRegionInfo, Result> splitParents =
771 new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
772 splitParents.put(parent, createResult(parent, splita, splitb));
773 splita.setOffline(true);
774 splitParents.put(splita, createResult(splita, splitaa,splitab));
775
776 final Map<HRegionInfo, Result> mergedRegions = new TreeMap<HRegionInfo, Result>();
777 CatalogJanitor janitor = spy(new CatalogJanitor(server, services));
778 doReturn(new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>(
779 10, mergedRegions, splitParents)).when(janitor)
780 .getMergedRegionsAndSplitParents();
781
782
783 Path splitaRef =
784 createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
785
786
787 assertEquals(0, janitor.scan());
788
789
790 FileSystem fs = FileSystem.get(htu.getConfiguration());
791 assertTrue(fs.delete(splitaRef, true));
792
793
794 assertEquals(2, janitor.scan());
795
796 services.stop("test finished");
797 janitor.cancel(true);
798 }
799
800
801
802
803
804 @Test
805 public void testSplitParentFirstComparator() {
806 SplitParentFirstComparator comp = new SplitParentFirstComparator();
807 final HTableDescriptor htd = createHTableDescriptor();
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828 HRegionInfo rootRegion = new HRegionInfo(htd.getTableName(),
829 HConstants.EMPTY_START_ROW,
830 HConstants.EMPTY_END_ROW, true);
831 HRegionInfo firstRegion = new HRegionInfo(htd.getTableName(),
832 HConstants.EMPTY_START_ROW,
833 Bytes.toBytes("bbb"), true);
834 HRegionInfo lastRegion = new HRegionInfo(htd.getTableName(),
835 Bytes.toBytes("bbb"),
836 HConstants.EMPTY_END_ROW, true);
837
838 assertTrue(comp.compare(rootRegion, rootRegion) == 0);
839 assertTrue(comp.compare(firstRegion, firstRegion) == 0);
840 assertTrue(comp.compare(lastRegion, lastRegion) == 0);
841 assertTrue(comp.compare(rootRegion, firstRegion) < 0);
842 assertTrue(comp.compare(rootRegion, lastRegion) < 0);
843 assertTrue(comp.compare(firstRegion, lastRegion) < 0);
844
845
846 HRegionInfo firstRegiona = new HRegionInfo(htd.getTableName(),
847 HConstants.EMPTY_START_ROW,
848 Bytes.toBytes("aaa"), true);
849 HRegionInfo firstRegionb = new HRegionInfo(htd.getTableName(),
850 Bytes.toBytes("aaa"),
851 Bytes.toBytes("bbb"), true);
852
853 HRegionInfo lastRegiona = new HRegionInfo(htd.getTableName(),
854 Bytes.toBytes("bbb"),
855 Bytes.toBytes("ddd"), true);
856 HRegionInfo lastRegionb = new HRegionInfo(htd.getTableName(),
857 Bytes.toBytes("ddd"),
858 HConstants.EMPTY_END_ROW, true);
859
860 assertTrue(comp.compare(firstRegiona, firstRegiona) == 0);
861 assertTrue(comp.compare(firstRegionb, firstRegionb) == 0);
862 assertTrue(comp.compare(rootRegion, firstRegiona) < 0);
863 assertTrue(comp.compare(rootRegion, firstRegionb) < 0);
864 assertTrue(comp.compare(firstRegion, firstRegiona) < 0);
865 assertTrue(comp.compare(firstRegion, firstRegionb) < 0);
866 assertTrue(comp.compare(firstRegiona, firstRegionb) < 0);
867
868 assertTrue(comp.compare(lastRegiona, lastRegiona) == 0);
869 assertTrue(comp.compare(lastRegionb, lastRegionb) == 0);
870 assertTrue(comp.compare(rootRegion, lastRegiona) < 0);
871 assertTrue(comp.compare(rootRegion, lastRegionb) < 0);
872 assertTrue(comp.compare(lastRegion, lastRegiona) < 0);
873 assertTrue(comp.compare(lastRegion, lastRegionb) < 0);
874 assertTrue(comp.compare(lastRegiona, lastRegionb) < 0);
875
876 assertTrue(comp.compare(firstRegiona, lastRegiona) < 0);
877 assertTrue(comp.compare(firstRegiona, lastRegionb) < 0);
878 assertTrue(comp.compare(firstRegionb, lastRegiona) < 0);
879 assertTrue(comp.compare(firstRegionb, lastRegionb) < 0);
880
881 HRegionInfo lastRegionaa = new HRegionInfo(htd.getTableName(),
882 Bytes.toBytes("bbb"),
883 Bytes.toBytes("ccc"), false);
884 HRegionInfo lastRegionab = new HRegionInfo(htd.getTableName(),
885 Bytes.toBytes("ccc"),
886 Bytes.toBytes("ddd"), false);
887
888 assertTrue(comp.compare(lastRegiona, lastRegionaa) < 0);
889 assertTrue(comp.compare(lastRegiona, lastRegionab) < 0);
890 assertTrue(comp.compare(lastRegionaa, lastRegionab) < 0);
891
892 }
893
894 @Test
895 public void testArchiveOldRegion() throws Exception {
896 String table = "table";
897 HBaseTestingUtility htu = new HBaseTestingUtility();
898 setRootDirAndCleanIt(htu, "testCleanParent");
899 Server server = new MockServer(htu);
900 MasterServices services = new MockMasterServices(server);
901
902
903 CatalogJanitor janitor = new CatalogJanitor(server, services);
904
905
906 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
907 htd.addFamily(new HColumnDescriptor("f"));
908 HRegionInfo parent = new HRegionInfo(htd.getTableName(),
909 Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
910 HRegionInfo splita = new HRegionInfo(htd.getTableName(),
911 Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
912 HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
913 Bytes.toBytes("ccc"),
914 Bytes.toBytes("eee"));
915
916
917
918 Result parentMetaRow = createResult(parent, splita, splitb);
919 FileSystem fs = FileSystem.get(htu.getConfiguration());
920 Path rootdir = services.getMasterFileSystem().getRootDir();
921
922
923
924 FSUtils.setRootDir(fs.getConf(), rootdir);
925 Path tabledir = FSUtils.getTableDir(rootdir, htd.getTableName());
926 Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
927 Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
928 tabledir, htd.getColumnFamilies()[0].getName());
929 LOG.debug("Table dir:" + tabledir);
930 LOG.debug("Store dir:" + storedir);
931 LOG.debug("Store archive dir:" + storeArchive);
932
933
934 FileStatus[] mockFiles = addMockStoreFiles(2, services, storedir);
935
936 FileStatus[] storeFiles = fs.listStatus(storedir);
937 int index = 0;
938 for (FileStatus file : storeFiles) {
939 LOG.debug("Have store file:" + file.getPath());
940 assertEquals("Got unexpected store file", mockFiles[index].getPath(),
941 storeFiles[index].getPath());
942 index++;
943 }
944
945
946 assertTrue(janitor.cleanParent(parent, parentMetaRow));
947 LOG.debug("Finished cleanup of parent region");
948
949
950 FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
951 logFiles("archived files", storeFiles);
952 logFiles("archived files", archivedStoreFiles);
953
954 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
955
956
957 FSUtils.delete(fs, rootdir, true);
958 services.stop("Test finished");
959 server.stop("Test finished");
960 janitor.cancel(true);
961 }
962
963
964
965
966
967 private void logFiles(String description, FileStatus[] storeFiles) {
968 LOG.debug("Current " + description + ": ");
969 for (FileStatus file : storeFiles) {
970 LOG.debug(file.getPath());
971 }
972 }
973
974
975
976
977
978 @Test
979 public void testDuplicateHFileResolution() throws Exception {
980 String table = "table";
981 HBaseTestingUtility htu = new HBaseTestingUtility();
982 setRootDirAndCleanIt(htu, "testCleanParent");
983 Server server = new MockServer(htu);
984 MasterServices services = new MockMasterServices(server);
985
986
987
988 CatalogJanitor janitor = new CatalogJanitor(server, services);
989
990
991 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
992 htd.addFamily(new HColumnDescriptor("f"));
993 HRegionInfo parent = new HRegionInfo(htd.getTableName(),
994 Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
995 HRegionInfo splita = new HRegionInfo(htd.getTableName(),
996 Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
997 HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
998 Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
999
1000
1001 Result r = createResult(parent, splita, splitb);
1002
1003 FileSystem fs = FileSystem.get(htu.getConfiguration());
1004
1005 Path rootdir = services.getMasterFileSystem().getRootDir();
1006
1007
1008
1009 FSUtils.setRootDir(fs.getConf(), rootdir);
1010 Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
1011 Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
1012 System.out.println("Old root:" + rootdir);
1013 System.out.println("Old table:" + tabledir);
1014 System.out.println("Old store:" + storedir);
1015
1016 Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
1017 tabledir, htd.getColumnFamilies()[0].getName());
1018 System.out.println("Old archive:" + storeArchive);
1019
1020
1021 addMockStoreFiles(2, services, storedir);
1022
1023 FileStatus[] storeFiles = fs.listStatus(storedir);
1024
1025 assertTrue(janitor.cleanParent(parent, r));
1026
1027
1028 FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
1029 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
1030
1031
1032
1033 addMockStoreFiles(2, services, storedir);
1034
1035
1036 assertTrue(janitor.cleanParent(parent, r));
1037
1038
1039 archivedStoreFiles = fs.listStatus(storeArchive);
1040 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs, true);
1041
1042
1043 services.stop("Test finished");
1044 server.stop("shutdown");
1045 janitor.cancel(true);
1046 }
1047
1048 private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir)
1049 throws IOException {
1050
1051 FileSystem fs = services.getMasterFileSystem().getFileSystem();
1052 fs.mkdirs(storedir);
1053
1054 for (int i = 0; i < count; i++) {
1055 Path storeFile = new Path(storedir, "_store" + i);
1056 FSDataOutputStream dos = fs.create(storeFile, true);
1057 dos.writeBytes("Some data: " + i);
1058 dos.close();
1059 }
1060 LOG.debug("Adding " + count + " store files to the storedir:" + storedir);
1061
1062 FileStatus[] storeFiles = fs.listStatus(storedir);
1063 assertEquals("Didn't have expected store files", count, storeFiles.length);
1064 return storeFiles;
1065 }
1066
1067 private String setRootDirAndCleanIt(final HBaseTestingUtility htu,
1068 final String subdir)
1069 throws IOException {
1070 Path testdir = htu.getDataTestDir(subdir);
1071 FileSystem fs = FileSystem.get(htu.getConfiguration());
1072 if (fs.exists(testdir)) assertTrue(fs.delete(testdir, true));
1073 FSUtils.setRootDir(htu.getConfiguration(), testdir);
1074 return FSUtils.getRootDir(htu.getConfiguration()).toString();
1075 }
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087 private Path createReferences(final MasterServices services,
1088 final HTableDescriptor htd, final HRegionInfo parent,
1089 final HRegionInfo daughter, final byte [] midkey, final boolean top)
1090 throws IOException {
1091 Path rootdir = services.getMasterFileSystem().getRootDir();
1092 Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
1093 Path storedir = HStore.getStoreHomedir(tabledir, daughter,
1094 htd.getColumnFamilies()[0].getName());
1095 Reference ref =
1096 top? Reference.createTopReference(midkey): Reference.createBottomReference(midkey);
1097 long now = System.currentTimeMillis();
1098
1099 Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
1100 FileSystem fs = services.getMasterFileSystem().getFileSystem();
1101 ref.write(fs, p);
1102 return p;
1103 }
1104
1105 private Result createResult(final HRegionInfo parent, final HRegionInfo a,
1106 final HRegionInfo b)
1107 throws IOException {
1108 return MetaMockingUtil.getMetaTableRowResult(parent, null, a, b);
1109 }
1110
1111 private HTableDescriptor createHTableDescriptor() {
1112 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("t"));
1113 htd.addFamily(new HColumnDescriptor("f"));
1114 return htd;
1115 }
1116
1117 private MultiResponse buildMultiResponse(MultiRequest req) {
1118 MultiResponse.Builder builder = MultiResponse.newBuilder();
1119 RegionActionResult.Builder regionActionResultBuilder =
1120 RegionActionResult.newBuilder();
1121 ResultOrException.Builder roeBuilder = ResultOrException.newBuilder();
1122 for (RegionAction regionAction: req.getRegionActionList()) {
1123 regionActionResultBuilder.clear();
1124 for (ClientProtos.Action action: regionAction.getActionList()) {
1125 roeBuilder.clear();
1126 roeBuilder.setResult(ClientProtos.Result.getDefaultInstance());
1127 roeBuilder.setIndex(action.getIndex());
1128 regionActionResultBuilder.addResultOrException(roeBuilder.build());
1129 }
1130 builder.addRegionActionResult(regionActionResultBuilder.build());
1131 }
1132 return builder.build();
1133 }
1134
1135 }
1136