1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase;
19
20 import com.google.common.annotations.VisibleForTesting;
21 import com.google.protobuf.ServiceException;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.hbase.classification.InterfaceAudience;
27 import org.apache.hadoop.hbase.client.ClusterConnection;
28 import org.apache.hadoop.hbase.client.Connection;
29 import org.apache.hadoop.hbase.client.ConnectionFactory;
30 import org.apache.hadoop.hbase.client.Delete;
31 import org.apache.hadoop.hbase.client.Get;
32 import org.apache.hadoop.hbase.client.HTable;
33 import org.apache.hadoop.hbase.client.Mutation;
34 import org.apache.hadoop.hbase.client.Put;
35 import org.apache.hadoop.hbase.client.RegionLocator;
36 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
37 import org.apache.hadoop.hbase.client.Result;
38 import org.apache.hadoop.hbase.client.ResultScanner;
39 import org.apache.hadoop.hbase.client.Scan;
40 import org.apache.hadoop.hbase.client.Table;
41 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
42 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
43 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
44 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
47 import org.apache.hadoop.hbase.util.Pair;
48 import org.apache.hadoop.hbase.util.PairOfSameType;
49 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
50 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
51
52 import java.io.IOException;
53 import java.io.InterruptedIOException;
54 import java.util.ArrayList;
55 import java.util.List;
56 import java.util.Map;
57 import java.util.NavigableMap;
58 import java.util.Set;
59 import java.util.SortedMap;
60 import java.util.TreeMap;
61 import java.util.regex.Matcher;
62 import java.util.regex.Pattern;
63
64
65
66
67
68
69
70
71
72
73 @InterfaceAudience.Private
74 public class MetaTableAccessor {
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112 private static final Log LOG = LogFactory.getLog(MetaTableAccessor.class);
113
114 static final byte [] META_REGION_PREFIX;
115 static {
116
117
118 int len = HRegionInfo.FIRST_META_REGIONINFO.getRegionName().length - 2;
119 META_REGION_PREFIX = new byte [len];
120 System.arraycopy(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), 0,
121 META_REGION_PREFIX, 0, len);
122 }
123
124
125 protected static final char META_REPLICA_ID_DELIMITER = '_';
126
127
128 private static final Pattern SERVER_COLUMN_PATTERN
129 = Pattern.compile("^server(_[0-9a-fA-F]{4})?$");
130
131
132
133
134
135
136
137
138
139
140 public static List<Result> fullScanOfMeta(Connection connection)
141 throws IOException {
142 CollectAllVisitor v = new CollectAllVisitor();
143 fullScan(connection, v, null);
144 return v.getResults();
145 }
146
147
148
149
150
151
152
153 public static void fullScan(Connection connection,
154 final Visitor visitor)
155 throws IOException {
156 fullScan(connection, visitor, null);
157 }
158
159
160
161
162
163
164
165 public static List<Result> fullScan(Connection connection)
166 throws IOException {
167 CollectAllVisitor v = new CollectAllVisitor();
168 fullScan(connection, v, null);
169 return v.getResults();
170 }
171
172
173
174
175
176
177
178 static Table getMetaHTable(final Connection connection)
179 throws IOException {
180
181 if (connection == null) {
182 throw new NullPointerException("No connection");
183 } else if (connection.isClosed()) {
184 throw new IOException("connection is closed");
185 }
186
187
188
189
190
191
192
193
194 if (connection instanceof ClusterConnection) {
195 if (((ClusterConnection) connection).isManaged()) {
196 return new HTable(TableName.META_TABLE_NAME, (ClusterConnection) connection);
197 }
198 }
199 return connection.getTable(TableName.META_TABLE_NAME);
200 }
201
202
203
204
205
206
207 private static Result get(final Table t, final Get g) throws IOException {
208 try {
209 return t.get(g);
210 } finally {
211 t.close();
212 }
213 }
214
215
216
217
218
219
220
221
222
223 @Deprecated
224 public static Pair<HRegionInfo, ServerName> getRegion(Connection connection, byte [] regionName)
225 throws IOException {
226 HRegionLocation location = getRegionLocation(connection, regionName);
227 return location == null
228 ? null
229 : new Pair<HRegionInfo, ServerName>(location.getRegionInfo(), location.getServerName());
230 }
231
232
233
234
235
236
237
238
239 public static HRegionLocation getRegionLocation(Connection connection,
240 byte[] regionName) throws IOException {
241 byte[] row = regionName;
242 HRegionInfo parsedInfo = null;
243 try {
244 parsedInfo = parseRegionInfoFromRegionName(regionName);
245 row = getMetaKeyForRegion(parsedInfo);
246 } catch (Exception parseEx) {
247
248 }
249 Get get = new Get(row);
250 get.addFamily(HConstants.CATALOG_FAMILY);
251 Result r = get(getMetaHTable(connection), get);
252 RegionLocations locations = getRegionLocations(r);
253 return locations == null
254 ? null
255 : locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId());
256 }
257
258
259
260
261
262
263
264
265 public static HRegionLocation getRegionLocation(Connection connection,
266 HRegionInfo regionInfo) throws IOException {
267 byte[] row = getMetaKeyForRegion(regionInfo);
268 Get get = new Get(row);
269 get.addFamily(HConstants.CATALOG_FAMILY);
270 Result r = get(getMetaHTable(connection), get);
271 return getRegionLocation(r, regionInfo, regionInfo.getReplicaId());
272 }
273
274
275 public static byte[] getMetaKeyForRegion(HRegionInfo regionInfo) {
276 return RegionReplicaUtil.getRegionInfoForDefaultReplica(regionInfo).getRegionName();
277 }
278
279
280
281
282
283 protected static HRegionInfo parseRegionInfoFromRegionName(byte[] regionName)
284 throws IOException {
285 byte[][] fields = HRegionInfo.parseRegionName(regionName);
286 long regionId = Long.parseLong(Bytes.toString(fields[2]));
287 int replicaId = fields.length > 3 ? Integer.parseInt(Bytes.toString(fields[3]), 16) : 0;
288 return new HRegionInfo(
289 TableName.valueOf(fields[0]), fields[1], fields[1], false, regionId, replicaId);
290 }
291
292
293
294
295
296
297
298
299 public static Result getRegionResult(Connection connection,
300 byte[] regionName) throws IOException {
301 Get get = new Get(regionName);
302 get.addFamily(HConstants.CATALOG_FAMILY);
303 return get(getMetaHTable(connection), get);
304 }
305
306
307
308
309
310
311 public static Pair<HRegionInfo, HRegionInfo> getRegionsFromMergeQualifier(
312 Connection connection, byte[] regionName) throws IOException {
313 Result result = getRegionResult(connection, regionName);
314 HRegionInfo mergeA = getHRegionInfo(result, HConstants.MERGEA_QUALIFIER);
315 HRegionInfo mergeB = getHRegionInfo(result, HConstants.MERGEB_QUALIFIER);
316 if (mergeA == null && mergeB == null) {
317 return null;
318 }
319 return new Pair<HRegionInfo, HRegionInfo>(mergeA, mergeB);
320 }
321
322
323
324
325
326
327
328
329
330 public static boolean tableExists(Connection connection,
331 final TableName tableName)
332 throws IOException {
333 if (tableName.equals(TableName.META_TABLE_NAME)) {
334
335 return true;
336 }
337
338 CollectingVisitor<HRegionInfo> visitor = new CollectingVisitor<HRegionInfo>() {
339 private HRegionInfo current = null;
340
341 @Override
342 public boolean visit(Result r) throws IOException {
343 RegionLocations locations = getRegionLocations(r);
344 if (locations == null || locations.getRegionLocation().getRegionInfo() == null) {
345 LOG.warn("No serialized HRegionInfo in " + r);
346 return true;
347 }
348 this.current = locations.getRegionLocation().getRegionInfo();
349 if (this.current == null) {
350 LOG.warn("No serialized HRegionInfo in " + r);
351 return true;
352 }
353 if (!isInsideTable(this.current, tableName)) return false;
354
355 super.visit(r);
356
357 return false;
358 }
359
360 @Override
361 void add(Result r) {
362
363 this.results.add(this.current);
364 }
365 };
366 fullScan(connection, visitor, getTableStartRowForMeta(tableName));
367
368 return visitor.getResults().size() >= 1;
369 }
370
371
372
373
374
375
376
377
378
379 public static List<HRegionInfo> getTableRegions(ZooKeeperWatcher zkw,
380 Connection connection, TableName tableName)
381 throws IOException {
382 return getTableRegions(zkw, connection, tableName, false);
383 }
384
385
386
387
388
389
390
391
392
393
394
395 public static List<HRegionInfo> getTableRegions(ZooKeeperWatcher zkw, Connection connection,
396 TableName tableName, final boolean excludeOfflinedSplitParents) throws IOException {
397 return getTableRegions(zkw, connection, tableName, excludeOfflinedSplitParents, false);
398 }
399
400
401
402
403
404
405
406
407
408
409
410 public static List<HRegionInfo> getTableRegions(ZooKeeperWatcher zkw, Connection connection,
411 TableName tableName, final boolean excludeOfflinedSplitParents,
412 final boolean excludeReplicaRegions) throws IOException {
413 List<Pair<HRegionInfo, ServerName>> result = null;
414 result = getTableRegionsAndLocations(zkw, connection, tableName, excludeOfflinedSplitParents,
415 excludeReplicaRegions);
416 return getListOfHRegionInfos(result);
417 }
418
419 static List<HRegionInfo> getListOfHRegionInfos(final List<Pair<HRegionInfo, ServerName>> pairs) {
420 if (pairs == null || pairs.isEmpty()) return null;
421 List<HRegionInfo> result = new ArrayList<HRegionInfo>(pairs.size());
422 for (Pair<HRegionInfo, ServerName> pair: pairs) {
423 result.add(pair.getFirst());
424 }
425 return result;
426 }
427
428
429
430
431
432
433
434 static boolean isInsideTable(final HRegionInfo current, final TableName tableName) {
435 return tableName.equals(current.getTable());
436 }
437
438
439
440
441
442
443 static byte [] getTableStartRowForMeta(TableName tableName) {
444 byte [] startRow = new byte[tableName.getName().length + 2];
445 System.arraycopy(tableName.getName(), 0, startRow, 0, tableName.getName().length);
446 startRow[startRow.length - 2] = HConstants.DELIMITER;
447 startRow[startRow.length - 1] = HConstants.DELIMITER;
448 return startRow;
449 }
450
451
452
453
454
455
456
457
458
459
460 public static Scan getScanForTableName(TableName tableName) {
461 String strName = tableName.getNameAsString();
462
463 byte[] startKey = Bytes.toBytes(strName + ",,");
464
465 byte[] stopKey = Bytes.toBytes(strName + " ,,");
466
467 Scan scan = new Scan(startKey);
468 scan.setStopRow(stopKey);
469 return scan;
470 }
471
472
473
474
475
476
477
478
479 public static List<Pair<HRegionInfo, ServerName>>
480 getTableRegionsAndLocations(ZooKeeperWatcher zkw,
481 Connection connection, TableName tableName)
482 throws IOException {
483 return getTableRegionsAndLocations(zkw, connection, tableName, true);
484 }
485
486
487
488
489
490
491
492
493 public static List<Pair<HRegionInfo, ServerName>> getTableRegionsAndLocations(
494 ZooKeeperWatcher zkw, Connection connection, final TableName tableName,
495 final boolean excludeOfflinedSplitParents) throws IOException {
496 return getTableRegionsAndLocations(zkw, connection, tableName, excludeOfflinedSplitParents,
497 false);
498 }
499
500
501
502
503
504
505
506
507
508 public static List<Pair<HRegionInfo, ServerName>> getTableRegionsAndLocations(
509 ZooKeeperWatcher zkw, Connection connection, final TableName tableName,
510 final boolean excludeOfflinedSplitParents, final boolean excludeReplicaRegions)
511 throws IOException {
512 if (tableName.equals(TableName.META_TABLE_NAME)) {
513
514 ServerName serverName = new MetaTableLocator().getMetaRegionLocation(zkw);
515 List<Pair<HRegionInfo, ServerName>> list =
516 new ArrayList<Pair<HRegionInfo, ServerName>>();
517 list.add(new Pair<HRegionInfo, ServerName>(HRegionInfo.FIRST_META_REGIONINFO,
518 serverName));
519 return list;
520 }
521
522 CollectingVisitor<Pair<HRegionInfo, ServerName>> visitor =
523 new CollectingVisitor<Pair<HRegionInfo, ServerName>>() {
524 private RegionLocations current = null;
525
526 @Override
527 public boolean visit(Result r) throws IOException {
528 current = getRegionLocations(r);
529 if (current == null || current.getRegionLocation().getRegionInfo() == null) {
530 LOG.warn("No serialized HRegionInfo in " + r);
531 return true;
532 }
533 HRegionInfo hri = current.getRegionLocation().getRegionInfo();
534 if (!isInsideTable(hri, tableName)) return false;
535 if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
536
537 return super.visit(r);
538 }
539
540 @Override
541 void add(Result r) {
542 if (current == null) {
543 return;
544 }
545 for (HRegionLocation loc : current.getRegionLocations()) {
546 if (loc != null) {
547
548 if (excludeReplicaRegions
549 && !RegionReplicaUtil.isDefaultReplica(loc.getRegionInfo())) {
550 continue;
551 }
552 this.results.add(new Pair<HRegionInfo, ServerName>(
553 loc.getRegionInfo(), loc.getServerName()));
554 }
555 }
556 }
557 };
558 fullScan(connection, visitor, getTableStartRowForMeta(tableName));
559 return visitor.getResults();
560 }
561
562
563
564
565
566
567
568
569 public static NavigableMap<HRegionInfo, Result>
570 getServerUserRegions(Connection connection, final ServerName serverName)
571 throws IOException {
572 final NavigableMap<HRegionInfo, Result> hris = new TreeMap<HRegionInfo, Result>();
573
574
575 CollectingVisitor<Result> v = new CollectingVisitor<Result>() {
576 @Override
577 void add(Result r) {
578 if (r == null || r.isEmpty()) return;
579 RegionLocations locations = getRegionLocations(r);
580 if (locations == null) return;
581 for (HRegionLocation loc : locations.getRegionLocations()) {
582 if (loc != null) {
583 if (loc.getServerName() != null && loc.getServerName().equals(serverName)) {
584 hris.put(loc.getRegionInfo(), r);
585 }
586 }
587 }
588 }
589 };
590 fullScan(connection, v);
591 return hris;
592 }
593
594 public static void fullScanMetaAndPrint(Connection connection)
595 throws IOException {
596 Visitor v = new Visitor() {
597 @Override
598 public boolean visit(Result r) throws IOException {
599 if (r == null || r.isEmpty()) return true;
600 LOG.info("fullScanMetaAndPrint.Current Meta Row: " + r);
601 RegionLocations locations = getRegionLocations(r);
602 if (locations == null) return true;
603 for (HRegionLocation loc : locations.getRegionLocations()) {
604 if (loc != null) {
605 LOG.info("fullScanMetaAndPrint.HRI Print= " + loc.getRegionInfo());
606 }
607 }
608 return true;
609 }
610 };
611 fullScan(connection, v);
612 }
613
614
615
616
617
618
619
620
621
622
623 public static void fullScan(Connection connection,
624 final Visitor visitor, final byte [] startrow)
625 throws IOException {
626 Scan scan = new Scan();
627 if (startrow != null) scan.setStartRow(startrow);
628 if (startrow == null) {
629 int caching = connection.getConfiguration()
630 .getInt(HConstants.HBASE_META_SCANNER_CACHING, 100);
631 scan.setCaching(caching);
632 }
633 scan.addFamily(HConstants.CATALOG_FAMILY);
634 Table metaTable = getMetaHTable(connection);
635 ResultScanner scanner = null;
636 try {
637 scanner = metaTable.getScanner(scan);
638 Result data;
639 while((data = scanner.next()) != null) {
640 if (data.isEmpty()) continue;
641
642 if (!visitor.visit(data)) break;
643 }
644 } finally {
645 if (scanner != null) scanner.close();
646 metaTable.close();
647 }
648 }
649
650
651
652
653
654 protected static byte[] getFamily() {
655 return HConstants.CATALOG_FAMILY;
656 }
657
658
659
660
661
662 protected static byte[] getRegionInfoColumn() {
663 return HConstants.REGIONINFO_QUALIFIER;
664 }
665
666
667
668
669
670
671 @VisibleForTesting
672 public static byte[] getServerColumn(int replicaId) {
673 return replicaId == 0
674 ? HConstants.SERVER_QUALIFIER
675 : Bytes.toBytes(HConstants.SERVER_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
676 + String.format(HRegionInfo.REPLICA_ID_FORMAT, replicaId));
677 }
678
679
680
681
682
683
684 @VisibleForTesting
685 public static byte[] getStartCodeColumn(int replicaId) {
686 return replicaId == 0
687 ? HConstants.STARTCODE_QUALIFIER
688 : Bytes.toBytes(HConstants.STARTCODE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
689 + String.format(HRegionInfo.REPLICA_ID_FORMAT, replicaId));
690 }
691
692
693
694
695
696
697 @VisibleForTesting
698 public static byte[] getSeqNumColumn(int replicaId) {
699 return replicaId == 0
700 ? HConstants.SEQNUM_QUALIFIER
701 : Bytes.toBytes(HConstants.SEQNUM_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
702 + String.format(HRegionInfo.REPLICA_ID_FORMAT, replicaId));
703 }
704
705
706
707
708
709
710
711 @VisibleForTesting
712 static int parseReplicaIdFromServerColumn(byte[] serverColumn) {
713 String serverStr = Bytes.toString(serverColumn);
714
715 Matcher matcher = SERVER_COLUMN_PATTERN.matcher(serverStr);
716 if (matcher.matches() && matcher.groupCount() > 0) {
717 String group = matcher.group(1);
718 if (group != null && group.length() > 0) {
719 return Integer.parseInt(group.substring(1), 16);
720 } else {
721 return 0;
722 }
723 }
724 return -1;
725 }
726
727
728
729
730
731
732 private static ServerName getServerName(final Result r, final int replicaId) {
733 byte[] serverColumn = getServerColumn(replicaId);
734 Cell cell = r.getColumnLatestCell(getFamily(), serverColumn);
735 if (cell == null || cell.getValueLength() == 0) return null;
736 String hostAndPort = Bytes.toString(
737 cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
738 byte[] startcodeColumn = getStartCodeColumn(replicaId);
739 cell = r.getColumnLatestCell(getFamily(), startcodeColumn);
740 if (cell == null || cell.getValueLength() == 0) return null;
741 return ServerName.valueOf(hostAndPort,
742 Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
743 }
744
745
746
747
748
749
750
751 private static long getSeqNumDuringOpen(final Result r, final int replicaId) {
752 Cell cell = r.getColumnLatestCell(getFamily(), getSeqNumColumn(replicaId));
753 if (cell == null || cell.getValueLength() == 0) return HConstants.NO_SEQNUM;
754 return Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
755 }
756
757
758
759
760
761
762 public static RegionLocations getRegionLocations(final Result r) {
763 if (r == null) return null;
764 HRegionInfo regionInfo = getHRegionInfo(r, getRegionInfoColumn());
765 if (regionInfo == null) return null;
766
767 List<HRegionLocation> locations = new ArrayList<HRegionLocation>(1);
768 NavigableMap<byte[],NavigableMap<byte[],byte[]>> familyMap = r.getNoVersionMap();
769
770 locations.add(getRegionLocation(r, regionInfo, regionInfo.getReplicaId()));
771
772 NavigableMap<byte[], byte[]> infoMap = familyMap.get(getFamily());
773 if (infoMap == null) return new RegionLocations(locations);
774
775
776 int replicaId = 0;
777 byte[] serverColumn = getServerColumn(replicaId);
778 SortedMap<byte[], byte[]> serverMap = null;
779 serverMap = infoMap.tailMap(serverColumn, false);
780
781 if (serverMap.isEmpty()) return new RegionLocations(locations);
782
783 for (Map.Entry<byte[], byte[]> entry : serverMap.entrySet()) {
784 replicaId = parseReplicaIdFromServerColumn(entry.getKey());
785 if (replicaId < 0) {
786 break;
787 }
788 HRegionLocation location = getRegionLocation(r, regionInfo, replicaId);
789
790
791 if (location == null || location.getServerName() == null) {
792 locations.add(null);
793 } else {
794 locations.add(location);
795 }
796 }
797
798 return new RegionLocations(locations);
799 }
800
801
802
803
804
805
806
807
808
809
810 private static HRegionLocation getRegionLocation(final Result r, final HRegionInfo regionInfo,
811 final int replicaId) {
812 ServerName serverName = getServerName(r, replicaId);
813 long seqNum = getSeqNumDuringOpen(r, replicaId);
814 HRegionInfo replicaInfo = RegionReplicaUtil.getRegionInfoForReplica(regionInfo, replicaId);
815 return new HRegionLocation(replicaInfo, serverName, seqNum);
816 }
817
818
819
820
821
822
823
824
825 public static HRegionInfo getHRegionInfo(Result data) {
826 return getHRegionInfo(data, HConstants.REGIONINFO_QUALIFIER);
827 }
828
829
830
831
832
833
834
835
836 private static HRegionInfo getHRegionInfo(final Result r, byte [] qualifier) {
837 Cell cell = r.getColumnLatestCell(getFamily(), qualifier);
838 if (cell == null) return null;
839 return HRegionInfo.parseFromOrNull(cell.getValueArray(),
840 cell.getValueOffset(), cell.getValueLength());
841 }
842
843
844
845
846
847
848
849
850 public static PairOfSameType<HRegionInfo> getDaughterRegions(Result data) {
851 HRegionInfo splitA = getHRegionInfo(data, HConstants.SPLITA_QUALIFIER);
852 HRegionInfo splitB = getHRegionInfo(data, HConstants.SPLITB_QUALIFIER);
853
854 return new PairOfSameType<HRegionInfo>(splitA, splitB);
855 }
856
857
858
859
860
861
862
863
864 public static PairOfSameType<HRegionInfo> getMergeRegions(Result data) {
865 HRegionInfo mergeA = getHRegionInfo(data, HConstants.MERGEA_QUALIFIER);
866 HRegionInfo mergeB = getHRegionInfo(data, HConstants.MERGEB_QUALIFIER);
867
868 return new PairOfSameType<HRegionInfo>(mergeA, mergeB);
869 }
870
871
872
873
874 public interface Visitor {
875
876
877
878
879
880
881 boolean visit(final Result r) throws IOException;
882 }
883
884
885
886
887 static abstract class CollectingVisitor<T> implements Visitor {
888 final List<T> results = new ArrayList<T>();
889 @Override
890 public boolean visit(Result r) throws IOException {
891 if (r == null || r.isEmpty()) return true;
892 add(r);
893 return true;
894 }
895
896 abstract void add(Result r);
897
898
899
900
901
902 List<T> getResults() {
903 return this.results;
904 }
905 }
906
907
908
909
910 static class CollectAllVisitor extends CollectingVisitor<Result> {
911 @Override
912 void add(Result r) {
913 this.results.add(r);
914 }
915 }
916
917
918
919
920
921
922
923
924 @Deprecated
925 public static int getRegionCount(final Configuration c, final String tableName)
926 throws IOException {
927 return getRegionCount(c, TableName.valueOf(tableName));
928 }
929
930
931
932
933
934
935
936
937 public static int getRegionCount(final Configuration c, final TableName tableName)
938 throws IOException {
939 try (Connection connection = ConnectionFactory.createConnection(c)) {
940 return getRegionCount(connection, tableName);
941 }
942 }
943
944
945
946
947
948
949
950
951 public static int getRegionCount(final Connection connection, final TableName tableName)
952 throws IOException {
953 try (RegionLocator locator = connection.getRegionLocator(tableName)) {
954 List<HRegionLocation> locations = locator.getAllRegionLocations();
955 return locations == null? 0: locations.size();
956 }
957 }
958
959
960
961
962
963
964
965
966 public static Put makePutFromRegionInfo(HRegionInfo regionInfo)
967 throws IOException {
968 return makePutFromRegionInfo(regionInfo, HConstants.LATEST_TIMESTAMP);
969 }
970
971
972
973 public static Put makePutFromRegionInfo(HRegionInfo regionInfo, long ts)
974 throws IOException {
975 Put put = new Put(regionInfo.getRegionName(), ts);
976 addRegionInfo(put, regionInfo);
977 return put;
978 }
979
980
981
982
983
984 public static Delete makeDeleteFromRegionInfo(HRegionInfo regionInfo) {
985 return makeDeleteFromRegionInfo(regionInfo, HConstants.LATEST_TIMESTAMP);
986 }
987
988
989
990
991
992 public static Delete makeDeleteFromRegionInfo(HRegionInfo regionInfo, long ts) {
993 if (regionInfo == null) {
994 throw new IllegalArgumentException("Can't make a delete for null region");
995 }
996 Delete delete = new Delete(regionInfo.getRegionName(), ts);
997 return delete;
998 }
999
1000
1001
1002
1003 public static Put addDaughtersToPut(Put put, HRegionInfo splitA, HRegionInfo splitB) {
1004 if (splitA != null) {
1005 put.addImmutable(
1006 HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER, splitA.toByteArray());
1007 }
1008 if (splitB != null) {
1009 put.addImmutable(
1010 HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER, splitB.toByteArray());
1011 }
1012 return put;
1013 }
1014
1015
1016
1017
1018
1019
1020
1021 static void putToMetaTable(final Connection connection, final Put p)
1022 throws IOException {
1023 put(getMetaHTable(connection), p);
1024 }
1025
1026
1027
1028
1029
1030
1031 private static void put(final Table t, final Put p) throws IOException {
1032 try {
1033 t.put(p);
1034 } finally {
1035 t.close();
1036 }
1037 }
1038
1039
1040
1041
1042
1043
1044
1045 public static void putsToMetaTable(final Connection connection, final List<Put> ps)
1046 throws IOException {
1047 Table t = getMetaHTable(connection);
1048 try {
1049 t.put(ps);
1050 } finally {
1051 t.close();
1052 }
1053 }
1054
1055
1056
1057
1058
1059
1060
1061 static void deleteFromMetaTable(final Connection connection, final Delete d)
1062 throws IOException {
1063 List<Delete> dels = new ArrayList<Delete>(1);
1064 dels.add(d);
1065 deleteFromMetaTable(connection, dels);
1066 }
1067
1068
1069
1070
1071
1072
1073
1074 public static void deleteFromMetaTable(final Connection connection, final List<Delete> deletes)
1075 throws IOException {
1076 Table t = getMetaHTable(connection);
1077 try {
1078 t.delete(deletes);
1079 } finally {
1080 t.close();
1081 }
1082 }
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092 public static void removeRegionReplicasFromMeta(Set<byte[]> metaRows,
1093 int replicaIndexToDeleteFrom, int numReplicasToRemove, Connection connection)
1094 throws IOException {
1095 int absoluteIndex = replicaIndexToDeleteFrom + numReplicasToRemove;
1096 for (byte[] row : metaRows) {
1097 Delete deleteReplicaLocations = new Delete(row);
1098 for (int i = replicaIndexToDeleteFrom; i < absoluteIndex; i++) {
1099 deleteReplicaLocations.deleteColumns(HConstants.CATALOG_FAMILY,
1100 getServerColumn(i));
1101 deleteReplicaLocations.deleteColumns(HConstants.CATALOG_FAMILY,
1102 getSeqNumColumn(i));
1103 deleteReplicaLocations.deleteColumns(HConstants.CATALOG_FAMILY,
1104 getStartCodeColumn(i));
1105 }
1106 deleteFromMetaTable(connection, deleteReplicaLocations);
1107 }
1108 }
1109
1110
1111
1112
1113
1114
1115
1116 public static void mutateMetaTable(final Connection connection,
1117 final List<Mutation> mutations)
1118 throws IOException {
1119 Table t = getMetaHTable(connection);
1120 try {
1121 t.batch(mutations);
1122 } catch (InterruptedException e) {
1123 InterruptedIOException ie = new InterruptedIOException(e.getMessage());
1124 ie.initCause(e);
1125 throw ie;
1126 } finally {
1127 t.close();
1128 }
1129 }
1130
1131
1132
1133
1134
1135
1136
1137 public static void addRegionToMeta(Connection connection,
1138 HRegionInfo regionInfo)
1139 throws IOException {
1140 putToMetaTable(connection, makePutFromRegionInfo(regionInfo));
1141 LOG.info("Added " + regionInfo.getRegionNameAsString());
1142 }
1143
1144
1145
1146
1147
1148
1149
1150
1151 public static void addRegionToMeta(Table meta, HRegionInfo regionInfo) throws IOException {
1152 addRegionToMeta(meta, regionInfo, null, null);
1153 }
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167 public static void addRegionToMeta(Table meta, HRegionInfo regionInfo,
1168 HRegionInfo splitA, HRegionInfo splitB) throws IOException {
1169 Put put = makePutFromRegionInfo(regionInfo);
1170 addDaughtersToPut(put, splitA, splitB);
1171 meta.put(put);
1172 if (LOG.isDebugEnabled()) {
1173 LOG.debug("Added " + regionInfo.getRegionNameAsString());
1174 }
1175 }
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189 public static void addRegionToMeta(Connection connection, HRegionInfo regionInfo,
1190 HRegionInfo splitA, HRegionInfo splitB) throws IOException {
1191 Table meta = getMetaHTable(connection);
1192 try {
1193 addRegionToMeta(meta, regionInfo, splitA, splitB);
1194 } finally {
1195 meta.close();
1196 }
1197 }
1198
1199
1200
1201
1202
1203
1204
1205 public static void addRegionsToMeta(Connection connection,
1206 List<HRegionInfo> regionInfos, int regionReplication)
1207 throws IOException {
1208 addRegionsToMeta(connection, regionInfos, regionReplication, HConstants.LATEST_TIMESTAMP);
1209 }
1210
1211
1212
1213
1214
1215
1216
1217
1218 public static void addRegionsToMeta(Connection connection,
1219 List<HRegionInfo> regionInfos, int regionReplication, long ts)
1220 throws IOException {
1221 List<Put> puts = new ArrayList<Put>();
1222 for (HRegionInfo regionInfo : regionInfos) {
1223 if (RegionReplicaUtil.isDefaultReplica(regionInfo)) {
1224 Put put = makePutFromRegionInfo(regionInfo, ts);
1225
1226
1227 for (int i = 1; i < regionReplication; i++) {
1228 addEmptyLocation(put, i);
1229 }
1230 puts.add(put);
1231 }
1232 }
1233 putsToMetaTable(connection, puts);
1234 LOG.info("Added " + puts.size());
1235 }
1236
1237
1238
1239
1240
1241
1242
1243 public static void addDaughter(final Connection connection,
1244 final HRegionInfo regionInfo, final ServerName sn, final long openSeqNum)
1245 throws NotAllMetaRegionsOnlineException, IOException {
1246 Put put = new Put(regionInfo.getRegionName());
1247 addRegionInfo(put, regionInfo);
1248 if (sn != null) {
1249 addLocation(put, sn, openSeqNum, -1, regionInfo.getReplicaId());
1250 }
1251 putToMetaTable(connection, put);
1252 LOG.info("Added daughter " + regionInfo.getEncodedName() +
1253 (sn == null? ", serverName=null": ", serverName=" + sn.toString()));
1254 }
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268 public static void mergeRegions(final Connection connection, HRegionInfo mergedRegion,
1269 HRegionInfo regionA, HRegionInfo regionB, ServerName sn, int regionReplication,
1270 long masterSystemTime)
1271 throws IOException {
1272 Table meta = getMetaHTable(connection);
1273 try {
1274 HRegionInfo copyOfMerged = new HRegionInfo(mergedRegion);
1275
1276
1277 long time = Math.max(EnvironmentEdgeManager.currentTime(), masterSystemTime);
1278
1279
1280 Put putOfMerged = makePutFromRegionInfo(copyOfMerged, time);
1281 putOfMerged.addImmutable(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER,
1282 regionA.toByteArray());
1283 putOfMerged.addImmutable(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER,
1284 regionB.toByteArray());
1285
1286
1287 Delete deleteA = makeDeleteFromRegionInfo(regionA, time);
1288 Delete deleteB = makeDeleteFromRegionInfo(regionB, time);
1289
1290
1291 addLocation(putOfMerged, sn, 1, -1, mergedRegion.getReplicaId());
1292
1293
1294
1295 for (int i = 1; i < regionReplication; i++) {
1296 addEmptyLocation(putOfMerged, i);
1297 }
1298
1299 byte[] tableRow = Bytes.toBytes(mergedRegion.getRegionNameAsString()
1300 + HConstants.DELIMITER);
1301 multiMutate(meta, tableRow, putOfMerged, deleteA, deleteB);
1302 } finally {
1303 meta.close();
1304 }
1305 }
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318 public static void splitRegion(final Connection connection,
1319 HRegionInfo parent, HRegionInfo splitA, HRegionInfo splitB,
1320 ServerName sn, int regionReplication) throws IOException {
1321 Table meta = getMetaHTable(connection);
1322 try {
1323 HRegionInfo copyOfParent = new HRegionInfo(parent);
1324 copyOfParent.setOffline(true);
1325 copyOfParent.setSplit(true);
1326
1327
1328 Put putParent = makePutFromRegionInfo(copyOfParent);
1329 addDaughtersToPut(putParent, splitA, splitB);
1330
1331
1332 Put putA = makePutFromRegionInfo(splitA);
1333 Put putB = makePutFromRegionInfo(splitB);
1334
1335 addSequenceNum(putA, 1, -1, splitA.getReplicaId());
1336 addSequenceNum(putB, 1, -1, splitB.getReplicaId());
1337
1338
1339
1340 for (int i = 1; i < regionReplication; i++) {
1341 addEmptyLocation(putA, i);
1342 addEmptyLocation(putB, i);
1343 }
1344
1345 byte[] tableRow = Bytes.toBytes(parent.getRegionNameAsString() + HConstants.DELIMITER);
1346 multiMutate(meta, tableRow, putParent, putA, putB);
1347 } finally {
1348 meta.close();
1349 }
1350 }
1351
1352
1353
1354
1355 private static void multiMutate(Table table, byte[] row, Mutation... mutations)
1356 throws IOException {
1357 CoprocessorRpcChannel channel = table.coprocessorService(row);
1358 MultiRowMutationProtos.MutateRowsRequest.Builder mmrBuilder
1359 = MultiRowMutationProtos.MutateRowsRequest.newBuilder();
1360 for (Mutation mutation : mutations) {
1361 if (mutation instanceof Put) {
1362 mmrBuilder.addMutationRequest(ProtobufUtil.toMutation(
1363 ClientProtos.MutationProto.MutationType.PUT, mutation));
1364 } else if (mutation instanceof Delete) {
1365 mmrBuilder.addMutationRequest(ProtobufUtil.toMutation(
1366 ClientProtos.MutationProto.MutationType.DELETE, mutation));
1367 } else {
1368 throw new DoNotRetryIOException("multi in MetaEditor doesn't support "
1369 + mutation.getClass().getName());
1370 }
1371 }
1372
1373 MultiRowMutationProtos.MultiRowMutationService.BlockingInterface service =
1374 MultiRowMutationProtos.MultiRowMutationService.newBlockingStub(channel);
1375 try {
1376 service.mutateRows(null, mmrBuilder.build());
1377 } catch (ServiceException ex) {
1378 ProtobufUtil.toIOException(ex);
1379 }
1380 }
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396 public static void updateRegionLocation(Connection connection,
1397 HRegionInfo regionInfo, ServerName sn, long openSeqNum,
1398 long masterSystemTime)
1399 throws IOException {
1400 updateLocation(connection, regionInfo, sn, openSeqNum, masterSystemTime);
1401 }
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417 private static void updateLocation(final Connection connection,
1418 HRegionInfo regionInfo, ServerName sn, long openSeqNum,
1419 long masterSystemTime)
1420 throws IOException {
1421
1422
1423 long time = Math.max(EnvironmentEdgeManager.currentTime(), masterSystemTime);
1424
1425
1426 Put put = new Put(getMetaKeyForRegion(regionInfo), time);
1427 HRegionInfo defaultRegionInfo = regionInfo;
1428 if (regionInfo.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1429 defaultRegionInfo = new HRegionInfo(regionInfo, HRegionInfo.DEFAULT_REPLICA_ID);
1430 }
1431 addRegionInfo(put, defaultRegionInfo);
1432 addLocation(put, sn, openSeqNum, time, regionInfo.getReplicaId());
1433 putToMetaTable(connection, put);
1434 LOG.info("Updated row " + defaultRegionInfo.getRegionNameAsString() +
1435 " with server=" + sn);
1436 }
1437
1438
1439
1440
1441
1442
1443
1444 public static void deleteRegion(Connection connection,
1445 HRegionInfo regionInfo)
1446 throws IOException {
1447 Delete delete = new Delete(regionInfo.getRegionName());
1448 deleteFromMetaTable(connection, delete);
1449 LOG.info("Deleted " + regionInfo.getRegionNameAsString());
1450 }
1451
1452
1453
1454
1455
1456
1457
1458
1459 public static void deleteRegions(Connection connection,
1460 List<HRegionInfo> regionsInfo, long ts) throws IOException {
1461 List<Delete> deletes = new ArrayList<Delete>(regionsInfo.size());
1462 for (HRegionInfo hri: regionsInfo) {
1463 deletes.add(new Delete(hri.getRegionName(), ts));
1464 }
1465 deleteFromMetaTable(connection, deletes);
1466 LOG.info("Deleted " + regionsInfo);
1467 }
1468
1469
1470
1471
1472
1473
1474 public static void deleteRegions(Connection connection,
1475 List<HRegionInfo> regionsInfo) throws IOException {
1476 deleteRegions(connection, regionsInfo, HConstants.LATEST_TIMESTAMP);
1477 }
1478
1479
1480
1481
1482
1483
1484
1485
1486 public static void mutateRegions(Connection connection,
1487 final List<HRegionInfo> regionsToRemove,
1488 final List<HRegionInfo> regionsToAdd)
1489 throws IOException {
1490 List<Mutation> mutation = new ArrayList<Mutation>();
1491 if (regionsToRemove != null) {
1492 for (HRegionInfo hri: regionsToRemove) {
1493 mutation.add(new Delete(hri.getRegionName()));
1494 }
1495 }
1496 if (regionsToAdd != null) {
1497 for (HRegionInfo hri: regionsToAdd) {
1498 mutation.add(makePutFromRegionInfo(hri));
1499 }
1500 }
1501 mutateMetaTable(connection, mutation);
1502 if (regionsToRemove != null && regionsToRemove.size() > 0) {
1503 LOG.debug("Deleted " + regionsToRemove);
1504 }
1505 if (regionsToAdd != null && regionsToAdd.size() > 0) {
1506 LOG.debug("Added " + regionsToAdd);
1507 }
1508 }
1509
1510
1511
1512
1513
1514
1515
1516 public static void overwriteRegions(Connection connection,
1517 List<HRegionInfo> regionInfos, int regionReplication) throws IOException {
1518
1519 long now = EnvironmentEdgeManager.currentTime();
1520 deleteRegions(connection, regionInfos, now);
1521
1522
1523
1524
1525
1526
1527 addRegionsToMeta(connection, regionInfos, regionReplication, now+1);
1528 LOG.info("Overwritten " + regionInfos);
1529 }
1530
1531
1532
1533
1534
1535
1536
1537 public static void deleteMergeQualifiers(Connection connection,
1538 final HRegionInfo mergedRegion) throws IOException {
1539 Delete delete = new Delete(mergedRegion.getRegionName());
1540 delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER);
1541 delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER);
1542 deleteFromMetaTable(connection, delete);
1543 LOG.info("Deleted references in merged region "
1544 + mergedRegion.getRegionNameAsString() + ", qualifier="
1545 + Bytes.toStringBinary(HConstants.MERGEA_QUALIFIER) + " and qualifier="
1546 + Bytes.toStringBinary(HConstants.MERGEB_QUALIFIER));
1547 }
1548
1549 private static Put addRegionInfo(final Put p, final HRegionInfo hri)
1550 throws IOException {
1551 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
1552 hri.toByteArray());
1553 return p;
1554 }
1555
1556 public static Put addLocation(final Put p, final ServerName sn, long openSeqNum,
1557 long time, int replicaId){
1558 if (time <= 0) {
1559 time = EnvironmentEdgeManager.currentTime();
1560 }
1561 p.addImmutable(HConstants.CATALOG_FAMILY, getServerColumn(replicaId), time,
1562 Bytes.toBytes(sn.getHostAndPort()));
1563 p.addImmutable(HConstants.CATALOG_FAMILY, getStartCodeColumn(replicaId), time,
1564 Bytes.toBytes(sn.getStartcode()));
1565 p.addImmutable(HConstants.CATALOG_FAMILY, getSeqNumColumn(replicaId), time,
1566 Bytes.toBytes(openSeqNum));
1567 return p;
1568 }
1569
1570 public static Put addEmptyLocation(final Put p, int replicaId) {
1571 long now = EnvironmentEdgeManager.currentTime();
1572 p.addImmutable(HConstants.CATALOG_FAMILY, getServerColumn(replicaId), now, null);
1573 p.addImmutable(HConstants.CATALOG_FAMILY, getStartCodeColumn(replicaId), now, null);
1574 p.addImmutable(HConstants.CATALOG_FAMILY, getSeqNumColumn(replicaId), now, null);
1575 return p;
1576 }
1577
1578 public static Put addSequenceNum(final Put p, long openSeqNum, long time, int replicaId) {
1579 if (time <= 0) {
1580 time = EnvironmentEdgeManager.currentTime();
1581 }
1582 p.addImmutable(HConstants.CATALOG_FAMILY, getSeqNumColumn(replicaId), time,
1583 Bytes.toBytes(openSeqNum));
1584 return p;
1585 }
1586
1587
1588
1589
1590
1591
1592
1593 public static boolean infoServerExists(Connection connection) throws IOException {
1594
1595 CollectingVisitor<Result> visitor = new CollectingVisitor<Result>() {
1596 @Override
1597 public boolean visit(Result r) throws IOException {
1598 if (r == null || r.isEmpty()) return true;
1599 RegionLocations locations = getRegionLocations(r);
1600 if (locations == null) return true;
1601 for (HRegionLocation loc : locations.getRegionLocations()) {
1602 if (loc != null) {
1603 if (loc.getServerName() != null) {
1604 add(r);
1605
1606 return false;
1607 }
1608 }
1609 }
1610 return true;
1611 }
1612
1613 @Override
1614 void add(Result r) {
1615 this.results.add(r);
1616 }
1617 };
1618 fullScan(connection, visitor);
1619
1620 return visitor.getResults().size() >= 1;
1621 }
1622 }