1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase;
19
20 import java.io.File;
21 import java.io.IOException;
22 import java.io.OutputStream;
23 import java.lang.reflect.Field;
24 import java.lang.reflect.Modifier;
25 import java.net.BindException;
26 import java.net.DatagramSocket;
27 import java.net.InetAddress;
28 import java.net.ServerSocket;
29 import java.net.Socket;
30 import java.net.UnknownHostException;
31 import java.security.MessageDigest;
32 import java.util.ArrayList;
33 import java.util.Arrays;
34 import java.util.Collection;
35 import java.util.Collections;
36 import java.util.HashSet;
37 import java.util.List;
38 import java.util.Map;
39 import java.util.NavigableSet;
40 import java.util.Properties;
41 import java.util.Random;
42 import java.util.Set;
43 import java.util.TreeSet;
44 import java.util.UUID;
45 import java.util.concurrent.TimeUnit;
46
47 import org.apache.commons.io.FileUtils;
48 import org.apache.commons.lang.RandomStringUtils;
49 import org.apache.commons.logging.Log;
50 import org.apache.commons.logging.LogFactory;
51 import org.apache.commons.logging.impl.Jdk14Logger;
52 import org.apache.commons.logging.impl.Log4JLogger;
53 import org.apache.hadoop.conf.Configuration;
54 import org.apache.hadoop.fs.FileSystem;
55 import org.apache.hadoop.fs.Path;
56 import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
57 import org.apache.hadoop.hbase.Waiter.Predicate;
58 import org.apache.hadoop.hbase.classification.InterfaceAudience;
59 import org.apache.hadoop.hbase.classification.InterfaceStability;
60 import org.apache.hadoop.hbase.client.Admin;
61 import org.apache.hadoop.hbase.client.Connection;
62 import org.apache.hadoop.hbase.client.ConnectionFactory;
63 import org.apache.hadoop.hbase.client.Consistency;
64 import org.apache.hadoop.hbase.client.Delete;
65 import org.apache.hadoop.hbase.client.Durability;
66 import org.apache.hadoop.hbase.client.Get;
67 import org.apache.hadoop.hbase.client.HBaseAdmin;
68 import org.apache.hadoop.hbase.client.HConnection;
69 import org.apache.hadoop.hbase.client.HTable;
70 import org.apache.hadoop.hbase.client.Put;
71 import org.apache.hadoop.hbase.client.RegionLocator;
72 import org.apache.hadoop.hbase.client.Result;
73 import org.apache.hadoop.hbase.client.ResultScanner;
74 import org.apache.hadoop.hbase.client.Scan;
75 import org.apache.hadoop.hbase.client.Table;
76 import org.apache.hadoop.hbase.fs.HFileSystem;
77 import org.apache.hadoop.hbase.io.compress.Compression;
78 import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
79 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
80 import org.apache.hadoop.hbase.io.hfile.ChecksumUtil;
81 import org.apache.hadoop.hbase.io.hfile.HFile;
82 import org.apache.hadoop.hbase.ipc.RpcServerInterface;
83 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
84 import org.apache.hadoop.hbase.mapreduce.MapreduceTestingShim;
85 import org.apache.hadoop.hbase.master.HMaster;
86 import org.apache.hadoop.hbase.master.AssignmentManager;
87 import org.apache.hadoop.hbase.master.RegionStates;
88 import org.apache.hadoop.hbase.master.ServerManager;
89 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
90 import org.apache.hadoop.hbase.regionserver.BloomType;
91 import org.apache.hadoop.hbase.regionserver.HRegion;
92 import org.apache.hadoop.hbase.regionserver.HRegionServer;
93 import org.apache.hadoop.hbase.regionserver.HStore;
94 import org.apache.hadoop.hbase.regionserver.InternalScanner;
95 import org.apache.hadoop.hbase.regionserver.Region;
96 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
97 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
98 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
99 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
100 import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
101 import org.apache.hadoop.hbase.security.User;
102 import org.apache.hadoop.hbase.security.visibility.VisibilityLabelsCache;
103 import org.apache.hadoop.hbase.tool.Canary;
104 import org.apache.hadoop.hbase.util.Bytes;
105 import org.apache.hadoop.hbase.util.FSTableDescriptors;
106 import org.apache.hadoop.hbase.util.FSUtils;
107 import org.apache.hadoop.hbase.util.JVMClusterUtil;
108 import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
109 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
110 import org.apache.hadoop.hbase.util.Pair;
111 import org.apache.hadoop.hbase.util.RegionSplitter;
112 import org.apache.hadoop.hbase.util.RetryCounter;
113 import org.apache.hadoop.hbase.util.Threads;
114 import org.apache.hadoop.hbase.wal.WAL;
115 import org.apache.hadoop.hbase.wal.WALFactory;
116 import org.apache.hadoop.hbase.zookeeper.EmptyWatcher;
117 import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
118 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
119 import org.apache.hadoop.hbase.zookeeper.ZKConfig;
120 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
121 import org.apache.hadoop.hdfs.DFSClient;
122 import org.apache.hadoop.hdfs.DistributedFileSystem;
123 import org.apache.hadoop.hdfs.MiniDFSCluster;
124 import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
125 import org.apache.hadoop.mapred.JobConf;
126 import org.apache.hadoop.mapred.MiniMRCluster;
127 import org.apache.hadoop.mapred.TaskLog;
128 import org.apache.hadoop.minikdc.MiniKdc;
129 import org.apache.zookeeper.KeeperException;
130 import org.apache.zookeeper.KeeperException.NodeExistsException;
131 import org.apache.zookeeper.WatchedEvent;
132 import org.apache.zookeeper.ZooKeeper;
133 import org.apache.zookeeper.ZooKeeper.States;
134
135 import static org.junit.Assert.assertEquals;
136 import static org.junit.Assert.assertTrue;
137 import static org.junit.Assert.fail;
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153 @InterfaceAudience.Public
154 @InterfaceStability.Evolving
155 @SuppressWarnings("deprecation")
156 public class HBaseTestingUtility extends HBaseCommonTestingUtility {
157 private MiniZooKeeperCluster zkCluster = null;
158
159 public static final String REGIONS_PER_SERVER_KEY = "hbase.test.regions-per-server";
160
161
162
163
164 public static final int DEFAULT_REGIONS_PER_SERVER = 3;
165
166
167 public static final String PRESPLIT_TEST_TABLE_KEY = "hbase.test.pre-split-table";
168 public static final boolean PRESPLIT_TEST_TABLE = true;
169
170 public static final String USE_LOCAL_FILESYSTEM = "hbase.test.local.fileSystem";
171
172
173
174
175 private boolean passedZkCluster = false;
176 private MiniDFSCluster dfsCluster = null;
177
178 private volatile HBaseCluster hbaseCluster = null;
179 private MiniMRCluster mrCluster = null;
180
181
182 private volatile boolean miniClusterRunning;
183
184 private String hadoopLogDir;
185
186
187 private File clusterTestDir = null;
188
189
190
191 private Path dataTestDirOnTestFS = null;
192
193
194
195
196 private volatile Connection connection;
197
198
199
200
201
202
203
204
205 @Deprecated
206 private static final String TEST_DIRECTORY_KEY = "test.build.data";
207
208
209 private static String FS_URI;
210
211
212 private static final Set<Integer> takenRandomPorts = new HashSet<Integer>();
213
214
215 public static final List<Object[]> COMPRESSION_ALGORITHMS_PARAMETERIZED =
216 Arrays.asList(new Object[][] {
217 { Compression.Algorithm.NONE },
218 { Compression.Algorithm.GZ }
219 });
220
221
222 public static final List<Object[]> BOOLEAN_PARAMETERIZED =
223 Arrays.asList(new Object[][] {
224 { new Boolean(false) },
225 { new Boolean(true) }
226 });
227
228
229 public static final List<Object[]> MEMSTORETS_TAGS_PARAMETRIZED = memStoreTSAndTagsCombination() ;
230
231 public static final Compression.Algorithm[] COMPRESSION_ALGORITHMS ={
232 Compression.Algorithm.NONE, Compression.Algorithm.GZ
233 };
234
235
236
237
238
239
240
241 public static boolean available(int port) {
242 ServerSocket ss = null;
243 DatagramSocket ds = null;
244 try {
245 ss = new ServerSocket(port);
246 ss.setReuseAddress(true);
247 ds = new DatagramSocket(port);
248 ds.setReuseAddress(true);
249 return true;
250 } catch (IOException e) {
251
252 } finally {
253 if (ds != null) {
254 ds.close();
255 }
256
257 if (ss != null) {
258 try {
259 ss.close();
260 } catch (IOException e) {
261
262 }
263 }
264 }
265
266 return false;
267 }
268
269
270
271
272
273 private static List<Object[]> bloomAndCompressionCombinations() {
274 List<Object[]> configurations = new ArrayList<Object[]>();
275 for (Compression.Algorithm comprAlgo :
276 HBaseTestingUtility.COMPRESSION_ALGORITHMS) {
277 for (BloomType bloomType : BloomType.values()) {
278 configurations.add(new Object[] { comprAlgo, bloomType });
279 }
280 }
281 return Collections.unmodifiableList(configurations);
282 }
283
284
285
286
287 private static List<Object[]> memStoreTSAndTagsCombination() {
288 List<Object[]> configurations = new ArrayList<Object[]>();
289 configurations.add(new Object[] { false, false });
290 configurations.add(new Object[] { false, true });
291 configurations.add(new Object[] { true, false });
292 configurations.add(new Object[] { true, true });
293 return Collections.unmodifiableList(configurations);
294 }
295
296 public static final Collection<Object[]> BLOOM_AND_COMPRESSION_COMBINATIONS =
297 bloomAndCompressionCombinations();
298
299 public HBaseTestingUtility() {
300 this(HBaseConfiguration.create());
301 }
302
303 public HBaseTestingUtility(Configuration conf) {
304 super(conf);
305
306
307 ChecksumUtil.generateExceptionForChecksumFailureForTest(true);
308 }
309
310
311
312
313
314
315
316 public static HBaseTestingUtility createLocalHTU() {
317 Configuration c = HBaseConfiguration.create();
318 return createLocalHTU(c);
319 }
320
321
322
323
324
325
326
327
328 public static HBaseTestingUtility createLocalHTU(Configuration c) {
329 HBaseTestingUtility htu = new HBaseTestingUtility(c);
330 String dataTestDir = htu.getDataTestDir().toString();
331 htu.getConfiguration().set(HConstants.HBASE_DIR, dataTestDir);
332 LOG.debug("Setting " + HConstants.HBASE_DIR + " to " + dataTestDir);
333 return htu;
334 }
335
336
337
338
339 public static void closeRegion(final Region r) throws IOException {
340 if (r != null) {
341 ((HRegion)r).close();
342 }
343 }
344
345
346
347
348
349
350
351
352
353
354
355
356 @Override
357 public Configuration getConfiguration() {
358 return super.getConfiguration();
359 }
360
361 public void setHBaseCluster(HBaseCluster hbaseCluster) {
362 this.hbaseCluster = hbaseCluster;
363 }
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381 @Override
382 protected Path setupDataTestDir() {
383 Path testPath = super.setupDataTestDir();
384 if (null == testPath) {
385 return null;
386 }
387
388 createSubDirAndSystemProperty(
389 "hadoop.log.dir",
390 testPath, "hadoop-log-dir");
391
392
393
394 createSubDirAndSystemProperty(
395 "hadoop.tmp.dir",
396 testPath, "hadoop-tmp-dir");
397
398
399 createSubDir(
400 "mapreduce.cluster.local.dir",
401 testPath, "mapred-local-dir");
402
403 return testPath;
404 }
405
406 public void setJobWithoutMRCluster() throws IOException {
407 conf.set("hbase.fs.tmp.dir", getDataTestDirOnTestFS("hbase-staging").toString());
408 conf.setBoolean(HBaseTestingUtility.USE_LOCAL_FILESYSTEM, true);
409 }
410
411 private void createSubDirAndSystemProperty(
412 String propertyName, Path parent, String subDirName){
413
414 String sysValue = System.getProperty(propertyName);
415
416 if (sysValue != null) {
417
418
419 LOG.info("System.getProperty(\""+propertyName+"\") already set to: "+
420 sysValue + " so I do NOT create it in " + parent);
421 String confValue = conf.get(propertyName);
422 if (confValue != null && !confValue.endsWith(sysValue)){
423 LOG.warn(
424 propertyName + " property value differs in configuration and system: "+
425 "Configuration="+confValue+" while System="+sysValue+
426 " Erasing configuration value by system value."
427 );
428 }
429 conf.set(propertyName, sysValue);
430 } else {
431
432 createSubDir(propertyName, parent, subDirName);
433 System.setProperty(propertyName, conf.get(propertyName));
434 }
435 }
436
437
438
439
440
441
442
443 private Path getBaseTestDirOnTestFS() throws IOException {
444 FileSystem fs = getTestFileSystem();
445 return new Path(fs.getWorkingDirectory(), "test-data");
446 }
447
448
449
450
451 public HTableDescriptor getMetaTableDescriptor() {
452 try {
453 return new FSTableDescriptors(conf).get(TableName.META_TABLE_NAME);
454 } catch (IOException e) {
455 throw new RuntimeException("Unable to create META table descriptor", e);
456 }
457 }
458
459
460
461
462
463
464 Path getClusterTestDir() {
465 if (clusterTestDir == null){
466 setupClusterTestDir();
467 }
468 return new Path(clusterTestDir.getAbsolutePath());
469 }
470
471
472
473
474 private void setupClusterTestDir() {
475 if (clusterTestDir != null) {
476 return;
477 }
478
479
480
481 Path testDir = getDataTestDir("dfscluster_" + UUID.randomUUID().toString());
482 clusterTestDir = new File(testDir.toString()).getAbsoluteFile();
483
484 boolean b = deleteOnExit();
485 if (b) clusterTestDir.deleteOnExit();
486 conf.set(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
487 LOG.info("Created new mini-cluster data directory: " + clusterTestDir + ", deleteOnExit=" + b);
488 }
489
490
491
492
493
494
495
496 public Path getDataTestDirOnTestFS() throws IOException {
497 if (dataTestDirOnTestFS == null) {
498 setupDataTestDirOnTestFS();
499 }
500
501 return dataTestDirOnTestFS;
502 }
503
504
505
506
507
508
509
510
511 public Path getDataTestDirOnTestFS(final String subdirName) throws IOException {
512 return new Path(getDataTestDirOnTestFS(), subdirName);
513 }
514
515
516
517
518
519 private void setupDataTestDirOnTestFS() throws IOException {
520 if (dataTestDirOnTestFS != null) {
521 LOG.warn("Data test on test fs dir already setup in "
522 + dataTestDirOnTestFS.toString());
523 return;
524 }
525 dataTestDirOnTestFS = getNewDataTestDirOnTestFS();
526 }
527
528
529
530
531 private Path getNewDataTestDirOnTestFS() throws IOException {
532
533
534
535
536 FileSystem fs = getTestFileSystem();
537 Path newDataTestDir = null;
538 if (fs.getUri().getScheme().equals(FileSystem.getLocal(conf).getUri().getScheme())) {
539 File dataTestDir = new File(getDataTestDir().toString());
540 if (deleteOnExit()) dataTestDir.deleteOnExit();
541 newDataTestDir = new Path(dataTestDir.getAbsolutePath());
542 } else {
543 Path base = getBaseTestDirOnTestFS();
544 String randomStr = UUID.randomUUID().toString();
545 newDataTestDir = new Path(base, randomStr);
546 if (deleteOnExit()) fs.deleteOnExit(newDataTestDir);
547 }
548 return newDataTestDir;
549 }
550
551
552
553
554
555
556 public boolean cleanupDataTestDirOnTestFS() throws IOException {
557 boolean ret = getTestFileSystem().delete(dataTestDirOnTestFS, true);
558 if (ret)
559 dataTestDirOnTestFS = null;
560 return ret;
561 }
562
563
564
565
566
567
568 public boolean cleanupDataTestDirOnTestFS(String subdirName) throws IOException {
569 Path cpath = getDataTestDirOnTestFS(subdirName);
570 return getTestFileSystem().delete(cpath, true);
571 }
572
573
574
575
576
577
578
579
580 public MiniDFSCluster startMiniDFSCluster(int servers) throws Exception {
581 return startMiniDFSCluster(servers, null);
582 }
583
584
585
586
587
588
589
590
591
592
593
594
595 public MiniDFSCluster startMiniDFSCluster(final String hosts[])
596 throws Exception {
597 if ( hosts != null && hosts.length != 0) {
598 return startMiniDFSCluster(hosts.length, hosts);
599 } else {
600 return startMiniDFSCluster(1, null);
601 }
602 }
603
604
605
606
607
608
609
610
611
612
613 public MiniDFSCluster startMiniDFSCluster(int servers, final String hosts[])
614 throws Exception {
615 createDirsAndSetProperties();
616 EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
617
618
619 org.apache.log4j.Logger.getLogger(org.apache.hadoop.metrics2.util.MBeans.class).
620 setLevel(org.apache.log4j.Level.ERROR);
621 org.apache.log4j.Logger.getLogger(org.apache.hadoop.metrics2.impl.MetricsSystemImpl.class).
622 setLevel(org.apache.log4j.Level.ERROR);
623
624
625 this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true,
626 true, null, null, hosts, null);
627
628
629 setFs();
630
631
632 this.dfsCluster.waitClusterUp();
633
634
635 dataTestDirOnTestFS = null;
636
637 return this.dfsCluster;
638 }
639
640 private void setFs() throws IOException {
641 if(this.dfsCluster == null){
642 LOG.info("Skipping setting fs because dfsCluster is null");
643 return;
644 }
645 FileSystem fs = this.dfsCluster.getFileSystem();
646 FSUtils.setFsDefault(this.conf, new Path(fs.getUri()));
647 if (this.conf.getBoolean(USE_LOCAL_FILESYSTEM, false)) {
648 FSUtils.setFsDefault(this.conf, new Path("file:///"));
649 }
650 }
651
652 public MiniDFSCluster startMiniDFSCluster(int servers, final String racks[], String hosts[])
653 throws Exception {
654 createDirsAndSetProperties();
655 this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true,
656 true, null, racks, hosts, null);
657
658
659 FileSystem fs = this.dfsCluster.getFileSystem();
660 FSUtils.setFsDefault(this.conf, new Path(fs.getUri()));
661
662
663 this.dfsCluster.waitClusterUp();
664
665
666 dataTestDirOnTestFS = null;
667
668 return this.dfsCluster;
669 }
670
671 public MiniDFSCluster startMiniDFSClusterForTestWAL(int namenodePort) throws IOException {
672 createDirsAndSetProperties();
673 dfsCluster = new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null,
674 null, null, null);
675 return dfsCluster;
676 }
677
678
679 private void createDirsAndSetProperties() throws IOException {
680 setupClusterTestDir();
681 System.setProperty(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
682 createDirAndSetProperty("cache_data", "test.cache.data");
683 createDirAndSetProperty("hadoop_tmp", "hadoop.tmp.dir");
684 hadoopLogDir = createDirAndSetProperty("hadoop_logs", "hadoop.log.dir");
685 createDirAndSetProperty("mapred_local", "mapreduce.cluster.local.dir");
686 createDirAndSetProperty("mapred_temp", "mapreduce.cluster.temp.dir");
687 enableShortCircuit();
688
689 Path root = getDataTestDirOnTestFS("hadoop");
690 conf.set(MapreduceTestingShim.getMROutputDirProp(),
691 new Path(root, "mapred-output-dir").toString());
692 conf.set("mapreduce.jobtracker.system.dir", new Path(root, "mapred-system-dir").toString());
693 conf.set("mapreduce.jobtracker.staging.root.dir",
694 new Path(root, "mapreduce-jobtracker-staging-root-dir").toString());
695 conf.set("mapreduce.job.working.dir", new Path(root, "mapred-working-dir").toString());
696 }
697
698
699
700
701
702
703
704 public boolean isReadShortCircuitOn(){
705 final String propName = "hbase.tests.use.shortcircuit.reads";
706 String readOnProp = System.getProperty(propName);
707 if (readOnProp != null){
708 return Boolean.parseBoolean(readOnProp);
709 } else {
710 return conf.getBoolean(propName, false);
711 }
712 }
713
714
715
716
717 private void enableShortCircuit() {
718 if (isReadShortCircuitOn()) {
719 String curUser = System.getProperty("user.name");
720 LOG.info("read short circuit is ON for user " + curUser);
721
722 conf.set("dfs.block.local-path-access.user", curUser);
723
724 conf.setBoolean("dfs.client.read.shortcircuit", true);
725
726 conf.setBoolean("dfs.client.read.shortcircuit.skip.checksum", true);
727 } else {
728 LOG.info("read short circuit is OFF");
729 }
730 }
731
732 private String createDirAndSetProperty(final String relPath, String property) {
733 String path = getDataTestDir(relPath).toString();
734 System.setProperty(property, path);
735 conf.set(property, path);
736 new File(path).mkdirs();
737 LOG.info("Setting " + property + " to " + path + " in system properties and HBase conf");
738 return path;
739 }
740
741
742
743
744
745
746 public void shutdownMiniDFSCluster() throws IOException {
747 if (this.dfsCluster != null) {
748
749 this.dfsCluster.shutdown();
750 dfsCluster = null;
751 dataTestDirOnTestFS = null;
752 FSUtils.setFsDefault(this.conf, new Path("file:///"));
753 }
754 }
755
756
757
758
759
760
761
762
763 public MiniZooKeeperCluster startMiniZKCluster() throws Exception {
764 return startMiniZKCluster(1);
765 }
766
767
768
769
770
771
772
773
774
775 public MiniZooKeeperCluster startMiniZKCluster(
776 final int zooKeeperServerNum,
777 final int ... clientPortList)
778 throws Exception {
779 setupClusterTestDir();
780 return startMiniZKCluster(clusterTestDir, zooKeeperServerNum, clientPortList);
781 }
782
783 private MiniZooKeeperCluster startMiniZKCluster(final File dir)
784 throws Exception {
785 return startMiniZKCluster(dir, 1, null);
786 }
787
788
789
790
791
792 private MiniZooKeeperCluster startMiniZKCluster(final File dir,
793 final int zooKeeperServerNum,
794 final int [] clientPortList)
795 throws Exception {
796 if (this.zkCluster != null) {
797 throw new IOException("Cluster already running at " + dir);
798 }
799 this.passedZkCluster = false;
800 this.zkCluster = new MiniZooKeeperCluster(this.getConfiguration());
801 final int defPort = this.conf.getInt("test.hbase.zookeeper.property.clientPort", 0);
802 if (defPort > 0){
803
804 this.zkCluster.setDefaultClientPort(defPort);
805 }
806
807 if (clientPortList != null) {
808
809 int clientPortListSize = (clientPortList.length <= zooKeeperServerNum) ?
810 clientPortList.length : zooKeeperServerNum;
811 for (int i=0; i < clientPortListSize; i++) {
812 this.zkCluster.addClientPort(clientPortList[i]);
813 }
814 }
815 int clientPort = this.zkCluster.startup(dir,zooKeeperServerNum);
816 this.conf.set(HConstants.ZOOKEEPER_CLIENT_PORT,
817 Integer.toString(clientPort));
818 return this.zkCluster;
819 }
820
821
822
823
824
825
826
827 public void shutdownMiniZKCluster() throws IOException {
828 if (this.zkCluster != null) {
829 this.zkCluster.shutdown();
830 this.zkCluster = null;
831 }
832 }
833
834
835
836
837
838
839
840 public MiniHBaseCluster startMiniCluster() throws Exception {
841 return startMiniCluster(1, 1);
842 }
843
844
845
846
847
848
849
850
851
852 public MiniHBaseCluster startMiniCluster(final int numSlaves, boolean create)
853 throws Exception {
854 return startMiniCluster(1, numSlaves, create);
855 }
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870 public MiniHBaseCluster startMiniCluster(final int numSlaves)
871 throws Exception {
872 return startMiniCluster(1, numSlaves, false);
873 }
874
875
876
877
878
879
880
881
882 public MiniHBaseCluster startMiniCluster(final int numMasters,
883 final int numSlaves, boolean create)
884 throws Exception {
885 return startMiniCluster(numMasters, numSlaves, null, create);
886 }
887
888
889
890
891
892
893
894 public MiniHBaseCluster startMiniCluster(final int numMasters,
895 final int numSlaves)
896 throws Exception {
897 return startMiniCluster(numMasters, numSlaves, null, false);
898 }
899
900 public MiniHBaseCluster startMiniCluster(final int numMasters,
901 final int numSlaves, final String[] dataNodeHosts, boolean create)
902 throws Exception {
903 return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts,
904 null, null, create);
905 }
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931 public MiniHBaseCluster startMiniCluster(final int numMasters,
932 final int numSlaves, final String[] dataNodeHosts) throws Exception {
933 return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts,
934 null, null);
935 }
936
937
938
939
940
941 public MiniHBaseCluster startMiniCluster(final int numMasters,
942 final int numSlaves, final int numDataNodes) throws Exception {
943 return startMiniCluster(numMasters, numSlaves, numDataNodes, null, null, null);
944 }
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973 public MiniHBaseCluster startMiniCluster(final int numMasters,
974 final int numSlaves, final String[] dataNodeHosts, Class<? extends HMaster> masterClass,
975 Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
976 throws Exception {
977 return startMiniCluster(
978 numMasters, numSlaves, numSlaves, dataNodeHosts, masterClass, regionserverClass);
979 }
980
981 public MiniHBaseCluster startMiniCluster(final int numMasters,
982 final int numSlaves, int numDataNodes, final String[] dataNodeHosts,
983 Class<? extends HMaster> masterClass,
984 Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
985 throws Exception {
986 return startMiniCluster(numMasters, numSlaves, numDataNodes, dataNodeHosts,
987 masterClass, regionserverClass, false);
988 }
989
990
991
992
993
994
995
996
997 public MiniHBaseCluster startMiniCluster(final int numMasters,
998 final int numSlaves, int numDataNodes, final String[] dataNodeHosts,
999 Class<? extends HMaster> masterClass,
1000 Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass,
1001 boolean create)
1002 throws Exception {
1003 if (dataNodeHosts != null && dataNodeHosts.length != 0) {
1004 numDataNodes = dataNodeHosts.length;
1005 }
1006
1007 LOG.info("Starting up minicluster with " + numMasters + " master(s) and " +
1008 numSlaves + " regionserver(s) and " + numDataNodes + " datanode(s)");
1009
1010
1011 if (miniClusterRunning) {
1012 throw new IllegalStateException("A mini-cluster is already running");
1013 }
1014 miniClusterRunning = true;
1015
1016 setupClusterTestDir();
1017 System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestDir.getPath());
1018
1019
1020
1021 if(this.dfsCluster == null) {
1022 dfsCluster = startMiniDFSCluster(numDataNodes, dataNodeHosts);
1023 }
1024
1025
1026 if (this.zkCluster == null) {
1027 startMiniZKCluster(clusterTestDir);
1028 }
1029
1030
1031 return startMiniHBaseCluster(numMasters, numSlaves, masterClass,
1032 regionserverClass, create);
1033 }
1034
1035 public MiniHBaseCluster startMiniHBaseCluster(final int numMasters, final int numSlaves)
1036 throws IOException, InterruptedException{
1037 return startMiniHBaseCluster(numMasters, numSlaves, null, null, false);
1038 }
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053 public MiniHBaseCluster startMiniHBaseCluster(final int numMasters,
1054 final int numSlaves, Class<? extends HMaster> masterClass,
1055 Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass,
1056 boolean create)
1057 throws IOException, InterruptedException {
1058
1059 createRootDir(create);
1060
1061
1062
1063 if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1) == -1) {
1064 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, numSlaves);
1065 }
1066 if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1) == -1) {
1067 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, numSlaves);
1068 }
1069
1070 Configuration c = new Configuration(this.conf);
1071 this.hbaseCluster =
1072 new MiniHBaseCluster(c, numMasters, numSlaves, masterClass, regionserverClass);
1073
1074 Table t = new HTable(c, TableName.META_TABLE_NAME);
1075 ResultScanner s = t.getScanner(new Scan());
1076 while (s.next() != null) {
1077 continue;
1078 }
1079 s.close();
1080 t.close();
1081
1082 getHBaseAdmin();
1083 LOG.info("Minicluster is up");
1084
1085
1086
1087 setHBaseFsTmpDir();
1088
1089 return (MiniHBaseCluster)this.hbaseCluster;
1090 }
1091
1092
1093
1094
1095
1096
1097
1098 public void restartHBaseCluster(int servers) throws IOException, InterruptedException {
1099 this.hbaseCluster = new MiniHBaseCluster(this.conf, servers);
1100
1101 Table t = new HTable(new Configuration(this.conf), TableName.META_TABLE_NAME);
1102 ResultScanner s = t.getScanner(new Scan());
1103 while (s.next() != null) {
1104
1105 }
1106 LOG.info("HBase has been restarted");
1107 s.close();
1108 t.close();
1109 }
1110
1111
1112
1113
1114
1115
1116 public MiniHBaseCluster getMiniHBaseCluster() {
1117 if (this.hbaseCluster == null || this.hbaseCluster instanceof MiniHBaseCluster) {
1118 return (MiniHBaseCluster)this.hbaseCluster;
1119 }
1120 throw new RuntimeException(hbaseCluster + " not an instance of " +
1121 MiniHBaseCluster.class.getName());
1122 }
1123
1124
1125
1126
1127
1128
1129 public void shutdownMiniCluster() throws Exception {
1130 LOG.info("Shutting down minicluster");
1131 if (this.connection != null && !this.connection.isClosed()) {
1132 this.connection.close();
1133 this.connection = null;
1134 }
1135 shutdownMiniHBaseCluster();
1136 if (!this.passedZkCluster){
1137 shutdownMiniZKCluster();
1138 }
1139 shutdownMiniDFSCluster();
1140
1141 cleanupTestDir();
1142 miniClusterRunning = false;
1143 LOG.info("Minicluster is down");
1144 }
1145
1146
1147
1148
1149
1150 @Override
1151 public boolean cleanupTestDir() throws IOException {
1152 boolean ret = super.cleanupTestDir();
1153 if (deleteDir(this.clusterTestDir)) {
1154 this.clusterTestDir = null;
1155 return ret & true;
1156 }
1157 return false;
1158 }
1159
1160
1161
1162
1163
1164 public void shutdownMiniHBaseCluster() throws IOException {
1165 if (hbaseAdmin != null) {
1166 hbaseAdmin.close0();
1167 hbaseAdmin = null;
1168 }
1169
1170
1171 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1);
1172 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1);
1173 if (this.hbaseCluster != null) {
1174 this.hbaseCluster.shutdown();
1175
1176 this.hbaseCluster.waitUntilShutDown();
1177 this.hbaseCluster = null;
1178 }
1179
1180 if (zooKeeperWatcher != null) {
1181 zooKeeperWatcher.close();
1182 zooKeeperWatcher = null;
1183 }
1184 }
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194 public Path getDefaultRootDirPath(boolean create) throws IOException {
1195 if (!create) {
1196 return getDataTestDirOnTestFS();
1197 } else {
1198 return getNewDataTestDirOnTestFS();
1199 }
1200 }
1201
1202
1203
1204
1205
1206
1207
1208
1209 public Path getDefaultRootDirPath() throws IOException {
1210 return getDefaultRootDirPath(false);
1211 }
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225 public Path createRootDir(boolean create) throws IOException {
1226 FileSystem fs = FileSystem.get(this.conf);
1227 Path hbaseRootdir = getDefaultRootDirPath(create);
1228 FSUtils.setRootDir(this.conf, hbaseRootdir);
1229 fs.mkdirs(hbaseRootdir);
1230 FSUtils.setVersion(fs, hbaseRootdir);
1231 return hbaseRootdir;
1232 }
1233
1234
1235
1236
1237
1238
1239
1240 public Path createRootDir() throws IOException {
1241 return createRootDir(false);
1242 }
1243
1244
1245 private void setHBaseFsTmpDir() throws IOException {
1246 String hbaseFsTmpDirInString = this.conf.get("hbase.fs.tmp.dir");
1247 if (hbaseFsTmpDirInString == null) {
1248 this.conf.set("hbase.fs.tmp.dir", getDataTestDirOnTestFS("hbase-staging").toString());
1249 LOG.info("Setting hbase.fs.tmp.dir to " + this.conf.get("hbase.fs.tmp.dir"));
1250 } else {
1251 LOG.info("The hbase.fs.tmp.dir is set to " + hbaseFsTmpDirInString);
1252 }
1253 }
1254
1255
1256
1257
1258
1259 public void flush() throws IOException {
1260 getMiniHBaseCluster().flushcache();
1261 }
1262
1263
1264
1265
1266
1267 public void flush(TableName tableName) throws IOException {
1268 getMiniHBaseCluster().flushcache(tableName);
1269 }
1270
1271
1272
1273
1274
1275 public void compact(boolean major) throws IOException {
1276 getMiniHBaseCluster().compact(major);
1277 }
1278
1279
1280
1281
1282
1283 public void compact(TableName tableName, boolean major) throws IOException {
1284 getMiniHBaseCluster().compact(tableName, major);
1285 }
1286
1287
1288
1289
1290
1291
1292
1293
1294 public Table createTable(TableName tableName, String family)
1295 throws IOException{
1296 return createTable(tableName, new String[]{family});
1297 }
1298
1299
1300
1301
1302
1303
1304
1305
1306 public HTable createTable(byte[] tableName, byte[] family)
1307 throws IOException{
1308 return createTable(TableName.valueOf(tableName), new byte[][]{family});
1309 }
1310
1311
1312
1313
1314
1315
1316
1317
1318 public Table createTable(TableName tableName, String[] families)
1319 throws IOException {
1320 List<byte[]> fams = new ArrayList<byte[]>(families.length);
1321 for (String family : families) {
1322 fams.add(Bytes.toBytes(family));
1323 }
1324 return createTable(tableName, fams.toArray(new byte[0][]));
1325 }
1326
1327
1328
1329
1330
1331
1332
1333
1334 public HTable createTable(TableName tableName, byte[] family)
1335 throws IOException{
1336 return createTable(tableName, new byte[][]{family});
1337 }
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347 public HTable createMultiRegionTable(TableName tableName, byte[] family, int numRegions)
1348 throws IOException {
1349 if (numRegions < 3) throw new IOException("Must create at least 3 regions");
1350 byte[] startKey = Bytes.toBytes("aaaaa");
1351 byte[] endKey = Bytes.toBytes("zzzzz");
1352 byte[][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
1353
1354 return createTable(tableName, new byte[][] { family }, splitKeys);
1355 }
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365 public HTable createTable(byte[] tableName, byte[][] families)
1366 throws IOException {
1367 return createTable(tableName, families,
1368 new Configuration(getConfiguration()));
1369 }
1370
1371
1372
1373
1374
1375
1376
1377
1378 public HTable createTable(TableName tableName, byte[][] families)
1379 throws IOException {
1380 return createTable(tableName, families, (byte[][]) null);
1381 }
1382
1383
1384
1385
1386
1387
1388
1389
1390 public HTable createMultiRegionTable(TableName tableName, byte[][] families) throws IOException {
1391 return createTable(tableName, families, KEYS_FOR_HBA_CREATE_TABLE);
1392 }
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402 public HTable createTable(TableName tableName, byte[][] families, byte[][] splitKeys)
1403 throws IOException {
1404 return createTable(tableName, families, splitKeys, new Configuration(getConfiguration()));
1405 }
1406
1407 public HTable createTable(byte[] tableName, byte[][] families,
1408 int numVersions, byte[] startKey, byte[] endKey, int numRegions) throws IOException {
1409 return createTable(TableName.valueOf(tableName), families, numVersions,
1410 startKey, endKey, numRegions);
1411 }
1412
1413 public HTable createTable(String tableName, byte[][] families,
1414 int numVersions, byte[] startKey, byte[] endKey, int numRegions) throws IOException {
1415 return createTable(TableName.valueOf(tableName), families, numVersions,
1416 startKey, endKey, numRegions);
1417 }
1418
1419 public HTable createTable(TableName tableName, byte[][] families,
1420 int numVersions, byte[] startKey, byte[] endKey, int numRegions)
1421 throws IOException{
1422 HTableDescriptor desc = new HTableDescriptor(tableName);
1423 for (byte[] family : families) {
1424 HColumnDescriptor hcd = new HColumnDescriptor(family)
1425 .setMaxVersions(numVersions);
1426 desc.addFamily(hcd);
1427 }
1428 getHBaseAdmin().createTable(desc, startKey, endKey, numRegions);
1429
1430 waitUntilAllRegionsAssigned(tableName);
1431 return new HTable(getConfiguration(), tableName);
1432 }
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442 public HTable createTable(HTableDescriptor htd, byte[][] families, Configuration c)
1443 throws IOException {
1444 return createTable(htd, families, (byte[][]) null, c);
1445 }
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456 public HTable createTable(HTableDescriptor htd, byte[][] families, byte[][] splitKeys,
1457 Configuration c) throws IOException {
1458 for (byte[] family : families) {
1459 HColumnDescriptor hcd = new HColumnDescriptor(family);
1460
1461
1462
1463 hcd.setBloomFilterType(BloomType.NONE);
1464 htd.addFamily(hcd);
1465 }
1466 getHBaseAdmin().createTable(htd, splitKeys);
1467
1468
1469 waitUntilAllRegionsAssigned(htd.getTableName());
1470 return (HTable) getConnection().getTable(htd.getTableName());
1471 }
1472
1473
1474
1475
1476
1477
1478
1479
1480 public HTable createTable(HTableDescriptor htd, byte[][] splitRows)
1481 throws IOException {
1482 getHBaseAdmin().createTable(htd, splitRows);
1483
1484 waitUntilAllRegionsAssigned(htd.getTableName());
1485 return new HTable(getConfiguration(), htd.getTableName());
1486 }
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496 public HTable createTable(TableName tableName, byte[][] families,
1497 final Configuration c)
1498 throws IOException {
1499 return createTable(tableName, families, (byte[][]) null, c);
1500 }
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511 public HTable createTable(TableName tableName, byte[][] families, byte[][] splitKeys,
1512 final Configuration c) throws IOException {
1513 return createTable(new HTableDescriptor(tableName), families, splitKeys, c);
1514 }
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524 public HTable createTable(byte[] tableName, byte[][] families,
1525 final Configuration c)
1526 throws IOException {
1527 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
1528 for(byte[] family : families) {
1529 HColumnDescriptor hcd = new HColumnDescriptor(family);
1530
1531
1532
1533 hcd.setBloomFilterType(BloomType.NONE);
1534 desc.addFamily(hcd);
1535 }
1536 getHBaseAdmin().createTable(desc);
1537 return new HTable(c, desc.getTableName());
1538 }
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549 public HTable createTable(TableName tableName, byte[][] families,
1550 final Configuration c, int numVersions)
1551 throws IOException {
1552 HTableDescriptor desc = new HTableDescriptor(tableName);
1553 for(byte[] family : families) {
1554 HColumnDescriptor hcd = new HColumnDescriptor(family)
1555 .setMaxVersions(numVersions);
1556 desc.addFamily(hcd);
1557 }
1558 getHBaseAdmin().createTable(desc);
1559
1560 waitUntilAllRegionsAssigned(tableName);
1561 return new HTable(c, tableName);
1562 }
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573 public HTable createTable(byte[] tableName, byte[][] families,
1574 final Configuration c, int numVersions)
1575 throws IOException {
1576 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
1577 for(byte[] family : families) {
1578 HColumnDescriptor hcd = new HColumnDescriptor(family)
1579 .setMaxVersions(numVersions);
1580 desc.addFamily(hcd);
1581 }
1582 getHBaseAdmin().createTable(desc);
1583 return new HTable(c, desc.getTableName());
1584 }
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594 public HTable createTable(byte[] tableName, byte[] family, int numVersions)
1595 throws IOException {
1596 return createTable(tableName, new byte[][]{family}, numVersions);
1597 }
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607 public HTable createTable(TableName tableName, byte[] family, int numVersions)
1608 throws IOException {
1609 return createTable(tableName, new byte[][]{family}, numVersions);
1610 }
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620 public HTable createTable(byte[] tableName, byte[][] families,
1621 int numVersions)
1622 throws IOException {
1623 return createTable(TableName.valueOf(tableName), families, numVersions);
1624 }
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634 public HTable createTable(TableName tableName, byte[][] families,
1635 int numVersions)
1636 throws IOException {
1637 return createTable(tableName, families, numVersions, (byte[][]) null);
1638 }
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649 public HTable createTable(TableName tableName, byte[][] families, int numVersions,
1650 byte[][] splitKeys) throws IOException {
1651 HTableDescriptor desc = new HTableDescriptor(tableName);
1652 for (byte[] family : families) {
1653 HColumnDescriptor hcd = new HColumnDescriptor(family).setMaxVersions(numVersions);
1654 desc.addFamily(hcd);
1655 }
1656 getHBaseAdmin().createTable(desc, splitKeys);
1657
1658 waitUntilAllRegionsAssigned(tableName);
1659 return new HTable(new Configuration(getConfiguration()), tableName);
1660 }
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670 public HTable createMultiRegionTable(TableName tableName, byte[][] families, int numVersions)
1671 throws IOException {
1672 return createTable(tableName, families, numVersions, KEYS_FOR_HBA_CREATE_TABLE);
1673 }
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684 public HTable createTable(byte[] tableName, byte[][] families,
1685 int numVersions, int blockSize) throws IOException {
1686 return createTable(TableName.valueOf(tableName),
1687 families, numVersions, blockSize);
1688 }
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699 public HTable createTable(TableName tableName, byte[][] families,
1700 int numVersions, int blockSize) throws IOException {
1701 HTableDescriptor desc = new HTableDescriptor(tableName);
1702 for (byte[] family : families) {
1703 HColumnDescriptor hcd = new HColumnDescriptor(family)
1704 .setMaxVersions(numVersions)
1705 .setBlocksize(blockSize);
1706 desc.addFamily(hcd);
1707 }
1708 getHBaseAdmin().createTable(desc);
1709
1710 waitUntilAllRegionsAssigned(tableName);
1711 return new HTable(new Configuration(getConfiguration()), tableName);
1712 }
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722 public HTable createTable(byte[] tableName, byte[][] families,
1723 int[] numVersions)
1724 throws IOException {
1725 return createTable(TableName.valueOf(tableName), families, numVersions);
1726 }
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736 public HTable createTable(TableName tableName, byte[][] families,
1737 int[] numVersions)
1738 throws IOException {
1739 HTableDescriptor desc = new HTableDescriptor(tableName);
1740 int i = 0;
1741 for (byte[] family : families) {
1742 HColumnDescriptor hcd = new HColumnDescriptor(family)
1743 .setMaxVersions(numVersions[i]);
1744 desc.addFamily(hcd);
1745 i++;
1746 }
1747 getHBaseAdmin().createTable(desc);
1748
1749 waitUntilAllRegionsAssigned(tableName);
1750 return new HTable(new Configuration(getConfiguration()), tableName);
1751 }
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761 public HTable createTable(byte[] tableName, byte[] family, byte[][] splitRows)
1762 throws IOException{
1763 return createTable(TableName.valueOf(tableName), family, splitRows);
1764 }
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774 public HTable createTable(TableName tableName, byte[] family, byte[][] splitRows)
1775 throws IOException {
1776 HTableDescriptor desc = new HTableDescriptor(tableName);
1777 HColumnDescriptor hcd = new HColumnDescriptor(family);
1778 desc.addFamily(hcd);
1779 getHBaseAdmin().createTable(desc, splitRows);
1780
1781 waitUntilAllRegionsAssigned(tableName);
1782 return new HTable(getConfiguration(), tableName);
1783 }
1784
1785
1786
1787
1788
1789
1790
1791
1792 public HTable createMultiRegionTable(TableName tableName, byte[] family) throws IOException {
1793 return createTable(tableName, family, KEYS_FOR_HBA_CREATE_TABLE);
1794 }
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804 public HTable createTable(byte[] tableName, byte[][] families, byte[][] splitRows)
1805 throws IOException {
1806 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
1807 for(byte[] family:families) {
1808 HColumnDescriptor hcd = new HColumnDescriptor(family);
1809 desc.addFamily(hcd);
1810 }
1811 getHBaseAdmin().createTable(desc, splitRows);
1812
1813 waitUntilAllRegionsAssigned(desc.getTableName());
1814 return new HTable(getConfiguration(), desc.getTableName());
1815 }
1816
1817
1818
1819
1820 public static WAL createWal(final Configuration conf, final Path rootDir, final HRegionInfo hri)
1821 throws IOException {
1822
1823
1824 Configuration confForWAL = new Configuration(conf);
1825 confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
1826 return (new WALFactory(confForWAL,
1827 Collections.<WALActionsListener>singletonList(new MetricsWAL()),
1828 "hregion-" + RandomStringUtils.randomNumeric(8))).
1829 getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace());
1830 }
1831
1832
1833
1834
1835
1836 public static HRegion createRegionAndWAL(final HRegionInfo info, final Path rootDir,
1837 final Configuration conf, final HTableDescriptor htd) throws IOException {
1838 return createRegionAndWAL(info, rootDir, conf, htd, true);
1839 }
1840
1841
1842
1843
1844
1845 public static HRegion createRegionAndWAL(final HRegionInfo info, final Path rootDir,
1846 final Configuration conf, final HTableDescriptor htd, boolean initialize)
1847 throws IOException {
1848 WAL wal = createWal(conf, rootDir, info);
1849 return HRegion.createHRegion(info, rootDir, conf, htd, wal, initialize);
1850 }
1851
1852
1853
1854
1855 public static void closeRegionAndWAL(final Region r) throws IOException {
1856 closeRegionAndWAL((HRegion)r);
1857 }
1858
1859
1860
1861
1862 public static void closeRegionAndWAL(final HRegion r) throws IOException {
1863 if (r == null) return;
1864 r.close();
1865 if (r.getWAL() == null) return;
1866 r.getWAL().close();
1867 }
1868
1869
1870
1871
1872 @SuppressWarnings("serial")
1873 public static void modifyTableSync(Admin admin, HTableDescriptor desc)
1874 throws IOException, InterruptedException {
1875 admin.modifyTable(desc.getTableName(), desc);
1876 Pair<Integer, Integer> status = new Pair<Integer, Integer>() {{
1877 setFirst(0);
1878 setSecond(0);
1879 }};
1880 int i = 0;
1881 do {
1882 status = admin.getAlterStatus(desc.getTableName());
1883 if (status.getSecond() != 0) {
1884 LOG.debug(status.getSecond() - status.getFirst() + "/" + status.getSecond()
1885 + " regions updated.");
1886 Thread.sleep(1 * 1000l);
1887 } else {
1888 LOG.debug("All regions updated.");
1889 break;
1890 }
1891 } while (status.getFirst() != 0 && i++ < 500);
1892 if (status.getFirst() != 0) {
1893 throw new IOException("Failed to update all regions even after 500 seconds.");
1894 }
1895 }
1896
1897
1898
1899
1900 public static void setReplicas(Admin admin, TableName table, int replicaCount)
1901 throws IOException, InterruptedException {
1902 admin.disableTable(table);
1903 HTableDescriptor desc = admin.getTableDescriptor(table);
1904 desc.setRegionReplication(replicaCount);
1905 admin.modifyTable(desc.getTableName(), desc);
1906 admin.enableTable(table);
1907 }
1908
1909
1910
1911
1912
1913 public void deleteTable(String tableName) throws IOException {
1914 deleteTable(TableName.valueOf(tableName));
1915 }
1916
1917
1918
1919
1920
1921 public void deleteTable(byte[] tableName) throws IOException {
1922 deleteTable(TableName.valueOf(tableName));
1923 }
1924
1925
1926
1927
1928
1929 public void deleteTable(TableName tableName) throws IOException {
1930 try {
1931 getHBaseAdmin().disableTable(tableName);
1932 } catch (TableNotEnabledException e) {
1933 LOG.debug("Table: " + tableName + " already disabled, so just deleting it.");
1934 }
1935 getHBaseAdmin().deleteTable(tableName);
1936 }
1937
1938
1939
1940
1941
1942 public void deleteTableIfAny(TableName tableName) throws IOException {
1943 try {
1944 deleteTable(tableName);
1945 } catch (TableNotFoundException e) {
1946
1947 }
1948 }
1949
1950
1951
1952
1953
1954 public final static byte [] fam1 = Bytes.toBytes("colfamily11");
1955 public final static byte [] fam2 = Bytes.toBytes("colfamily21");
1956 public final static byte [] fam3 = Bytes.toBytes("colfamily31");
1957 public static final byte[][] COLUMNS = {fam1, fam2, fam3};
1958 private static final int MAXVERSIONS = 3;
1959
1960 public static final char FIRST_CHAR = 'a';
1961 public static final char LAST_CHAR = 'z';
1962 public static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
1963 public static final String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
1964
1965
1966
1967
1968
1969
1970
1971
1972 public HTableDescriptor createTableDescriptor(final String name,
1973 final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) {
1974 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
1975 for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
1976 htd.addFamily(new HColumnDescriptor(cfName)
1977 .setMinVersions(minVersions)
1978 .setMaxVersions(versions)
1979 .setKeepDeletedCells(keepDeleted)
1980 .setBlockCacheEnabled(false)
1981 .setTimeToLive(ttl)
1982 );
1983 }
1984 return htd;
1985 }
1986
1987
1988
1989
1990
1991
1992
1993 public HTableDescriptor createTableDescriptor(final String name) {
1994 return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
1995 MAXVERSIONS, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
1996 }
1997
1998
1999
2000
2001
2002 public HRegion createHRegion(
2003 final HRegionInfo info,
2004 final Path rootDir,
2005 final Configuration conf,
2006 final HTableDescriptor htd) throws IOException {
2007 return HRegion.createHRegion(info, rootDir, conf, htd);
2008 }
2009
2010 public HTableDescriptor createTableDescriptor(final TableName tableName,
2011 byte[] family) {
2012 return createTableDescriptor(tableName, new byte[][] {family}, 1);
2013 }
2014
2015 public HTableDescriptor createTableDescriptor(final TableName tableName,
2016 byte[][] families, int maxVersions) {
2017 HTableDescriptor desc = new HTableDescriptor(tableName);
2018 for (byte[] family : families) {
2019 HColumnDescriptor hcd = new HColumnDescriptor(family)
2020 .setMaxVersions(maxVersions);
2021 desc.addFamily(hcd);
2022 }
2023 return desc;
2024 }
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034 public HRegion createLocalHRegion(HTableDescriptor desc, byte [] startKey,
2035 byte [] endKey)
2036 throws IOException {
2037 HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
2038 return createLocalHRegion(hri, desc);
2039 }
2040
2041
2042
2043
2044
2045
2046
2047
2048 public HRegion createLocalHRegion(HRegionInfo info, HTableDescriptor desc) throws IOException {
2049 return HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), desc);
2050 }
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060 public HRegion createLocalHRegion(HRegionInfo info, HTableDescriptor desc, WAL wal)
2061 throws IOException {
2062 return HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), desc, wal);
2063 }
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077 public HRegion createLocalHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
2078 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
2079 WAL wal, byte[]... families) throws IOException {
2080 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
2081 htd.setReadOnly(isReadOnly);
2082 for (byte[] family : families) {
2083 HColumnDescriptor hcd = new HColumnDescriptor(family);
2084
2085 hcd.setMaxVersions(Integer.MAX_VALUE);
2086 htd.addFamily(hcd);
2087 }
2088 htd.setDurability(durability);
2089 HRegionInfo info = new HRegionInfo(htd.getTableName(), startKey, stopKey, false);
2090 return createLocalHRegion(info, htd, wal);
2091 }
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102 public HTable deleteTableData(byte[] tableName) throws IOException {
2103 return deleteTableData(TableName.valueOf(tableName));
2104 }
2105
2106
2107
2108
2109
2110
2111
2112
2113 public HTable deleteTableData(TableName tableName) throws IOException {
2114 HTable table = new HTable(getConfiguration(), tableName);
2115 Scan scan = new Scan();
2116 ResultScanner resScan = table.getScanner(scan);
2117 for(Result res : resScan) {
2118 Delete del = new Delete(res.getRow());
2119 table.delete(del);
2120 }
2121 resScan = table.getScanner(scan);
2122 resScan.close();
2123 return table;
2124 }
2125
2126
2127
2128
2129
2130
2131
2132
2133 public HTable truncateTable(final TableName tableName, final boolean preserveRegions)
2134 throws IOException {
2135 Admin admin = getHBaseAdmin();
2136 if (!admin.isTableDisabled(tableName)) {
2137 admin.disableTable(tableName);
2138 }
2139 admin.truncateTable(tableName, preserveRegions);
2140 return new HTable(getConfiguration(), tableName);
2141 }
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152 public HTable truncateTable(final TableName tableName) throws IOException {
2153 return truncateTable(tableName, false);
2154 }
2155
2156
2157
2158
2159
2160
2161
2162
2163 public HTable truncateTable(final byte[] tableName, final boolean preserveRegions)
2164 throws IOException {
2165 return truncateTable(TableName.valueOf(tableName), preserveRegions);
2166 }
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177 public HTable truncateTable(final byte[] tableName) throws IOException {
2178 return truncateTable(tableName, false);
2179 }
2180
2181
2182
2183
2184
2185
2186
2187
2188 public int loadTable(final Table t, final byte[] f) throws IOException {
2189 return loadTable(t, new byte[][] {f});
2190 }
2191
2192
2193
2194
2195
2196
2197
2198
2199 public int loadTable(final Table t, final byte[] f, boolean writeToWAL) throws IOException {
2200 return loadTable(t, new byte[][] {f}, null, writeToWAL);
2201 }
2202
2203
2204
2205
2206
2207
2208
2209
2210 public int loadTable(final Table t, final byte[][] f) throws IOException {
2211 return loadTable(t, f, null);
2212 }
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222 public int loadTable(final Table t, final byte[][] f, byte[] value) throws IOException {
2223 return loadTable(t, f, value, true);
2224 }
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234 public int loadTable(final Table t, final byte[][] f, byte[] value, boolean writeToWAL) throws IOException {
2235 List<Put> puts = new ArrayList<>();
2236 for (byte[] row : HBaseTestingUtility.ROWS) {
2237 Put put = new Put(row);
2238 put.setDurability(writeToWAL ? Durability.USE_DEFAULT : Durability.SKIP_WAL);
2239 for (int i = 0; i < f.length; i++) {
2240 put.add(f[i], f[i], value != null ? value : row);
2241 }
2242 puts.add(put);
2243 }
2244 t.put(puts);
2245 return puts.size();
2246 }
2247
2248
2249
2250
2251 public static class SeenRowTracker {
2252 int dim = 'z' - 'a' + 1;
2253 int[][][] seenRows = new int[dim][dim][dim];
2254 byte[] startRow;
2255 byte[] stopRow;
2256
2257 public SeenRowTracker(byte[] startRow, byte[] stopRow) {
2258 this.startRow = startRow;
2259 this.stopRow = stopRow;
2260 }
2261
2262 void reset() {
2263 for (byte[] row : ROWS) {
2264 seenRows[i(row[0])][i(row[1])][i(row[2])] = 0;
2265 }
2266 }
2267
2268 int i(byte b) {
2269 return b - 'a';
2270 }
2271
2272 public void addRow(byte[] row) {
2273 seenRows[i(row[0])][i(row[1])][i(row[2])]++;
2274 }
2275
2276
2277
2278
2279 public void validate() {
2280 for (byte b1 = 'a'; b1 <= 'z'; b1++) {
2281 for (byte b2 = 'a'; b2 <= 'z'; b2++) {
2282 for (byte b3 = 'a'; b3 <= 'z'; b3++) {
2283 int count = seenRows[i(b1)][i(b2)][i(b3)];
2284 int expectedCount = 0;
2285 if (Bytes.compareTo(new byte[] {b1,b2,b3}, startRow) >= 0
2286 && Bytes.compareTo(new byte[] {b1,b2,b3}, stopRow) < 0) {
2287 expectedCount = 1;
2288 }
2289 if (count != expectedCount) {
2290 String row = new String(new byte[] {b1,b2,b3});
2291 throw new RuntimeException("Row:" + row + " has a seen count of " + count + " instead of " + expectedCount);
2292 }
2293 }
2294 }
2295 }
2296 }
2297 }
2298
2299 public int loadRegion(final HRegion r, final byte[] f) throws IOException {
2300 return loadRegion(r, f, false);
2301 }
2302
2303 public int loadRegion(final Region r, final byte[] f) throws IOException {
2304 return loadRegion((HRegion)r, f);
2305 }
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315 public int loadRegion(final HRegion r, final byte[] f, final boolean flush)
2316 throws IOException {
2317 byte[] k = new byte[3];
2318 int rowCount = 0;
2319 for (byte b1 = 'a'; b1 <= 'z'; b1++) {
2320 for (byte b2 = 'a'; b2 <= 'z'; b2++) {
2321 for (byte b3 = 'a'; b3 <= 'z'; b3++) {
2322 k[0] = b1;
2323 k[1] = b2;
2324 k[2] = b3;
2325 Put put = new Put(k);
2326 put.setDurability(Durability.SKIP_WAL);
2327 put.add(f, null, k);
2328 if (r.getWAL() == null) {
2329 put.setDurability(Durability.SKIP_WAL);
2330 }
2331 int preRowCount = rowCount;
2332 int pause = 10;
2333 int maxPause = 1000;
2334 while (rowCount == preRowCount) {
2335 try {
2336 r.put(put);
2337 rowCount++;
2338 } catch (RegionTooBusyException e) {
2339 pause = (pause * 2 >= maxPause) ? maxPause : pause * 2;
2340 Threads.sleep(pause);
2341 }
2342 }
2343 }
2344 }
2345 if (flush) {
2346 r.flush(true);
2347 }
2348 }
2349 return rowCount;
2350 }
2351
2352 public void loadNumericRows(final Table t, final byte[] f, int startRow, int endRow)
2353 throws IOException {
2354 for (int i = startRow; i < endRow; i++) {
2355 byte[] data = Bytes.toBytes(String.valueOf(i));
2356 Put put = new Put(data);
2357 put.add(f, null, data);
2358 t.put(put);
2359 }
2360 }
2361
2362 public void verifyNumericRows(Table table, final byte[] f, int startRow, int endRow,
2363 int replicaId)
2364 throws IOException {
2365 for (int i = startRow; i < endRow; i++) {
2366 String failMsg = "Failed verification of row :" + i;
2367 byte[] data = Bytes.toBytes(String.valueOf(i));
2368 Get get = new Get(data);
2369 get.setReplicaId(replicaId);
2370 get.setConsistency(Consistency.TIMELINE);
2371 Result result = table.get(get);
2372 assertTrue(failMsg, result.containsColumn(f, null));
2373 assertEquals(failMsg, result.getColumnCells(f, null).size(), 1);
2374 Cell cell = result.getColumnLatestCell(f, null);
2375 assertTrue(failMsg,
2376 Bytes.equals(data, 0, data.length, cell.getValueArray(), cell.getValueOffset(),
2377 cell.getValueLength()));
2378 }
2379 }
2380
2381 public void verifyNumericRows(Region region, final byte[] f, int startRow, int endRow)
2382 throws IOException {
2383 verifyNumericRows((HRegion)region, f, startRow, endRow);
2384 }
2385
2386 public void verifyNumericRows(HRegion region, final byte[] f, int startRow, int endRow)
2387 throws IOException {
2388 verifyNumericRows(region, f, startRow, endRow, true);
2389 }
2390
2391 public void verifyNumericRows(Region region, final byte[] f, int startRow, int endRow,
2392 final boolean present) throws IOException {
2393 verifyNumericRows((HRegion)region, f, startRow, endRow, present);
2394 }
2395
2396 public void verifyNumericRows(HRegion region, final byte[] f, int startRow, int endRow,
2397 final boolean present) throws IOException {
2398 for (int i = startRow; i < endRow; i++) {
2399 String failMsg = "Failed verification of row :" + i;
2400 byte[] data = Bytes.toBytes(String.valueOf(i));
2401 Result result = region.get(new Get(data));
2402
2403 boolean hasResult = result != null && !result.isEmpty();
2404 assertEquals(failMsg + result, present, hasResult);
2405 if (!present) continue;
2406
2407 assertTrue(failMsg, result.containsColumn(f, null));
2408 assertEquals(failMsg, result.getColumnCells(f, null).size(), 1);
2409 Cell cell = result.getColumnLatestCell(f, null);
2410 assertTrue(failMsg,
2411 Bytes.equals(data, 0, data.length, cell.getValueArray(), cell.getValueOffset(),
2412 cell.getValueLength()));
2413 }
2414 }
2415
2416 public void deleteNumericRows(final HTable t, final byte[] f, int startRow, int endRow)
2417 throws IOException {
2418 for (int i = startRow; i < endRow; i++) {
2419 byte[] data = Bytes.toBytes(String.valueOf(i));
2420 Delete delete = new Delete(data);
2421 delete.deleteFamily(f);
2422 t.delete(delete);
2423 }
2424 }
2425
2426
2427
2428
2429 public int countRows(final Table table) throws IOException {
2430 Scan scan = new Scan();
2431 ResultScanner results = table.getScanner(scan);
2432 int count = 0;
2433 for (@SuppressWarnings("unused") Result res : results) {
2434 count++;
2435 }
2436 results.close();
2437 return count;
2438 }
2439
2440 public int countRows(final Table table, final byte[]... families) throws IOException {
2441 Scan scan = new Scan();
2442 for (byte[] family: families) {
2443 scan.addFamily(family);
2444 }
2445 ResultScanner results = table.getScanner(scan);
2446 int count = 0;
2447 for (@SuppressWarnings("unused") Result res : results) {
2448 count++;
2449 }
2450 results.close();
2451 return count;
2452 }
2453
2454
2455
2456
2457 public int countRows(final TableName tableName) throws IOException {
2458 Table table = getConnection().getTable(tableName);
2459 try {
2460 return countRows(table);
2461 } finally {
2462 table.close();
2463 }
2464 }
2465
2466
2467
2468
2469 public String checksumRows(final Table table) throws Exception {
2470 Scan scan = new Scan();
2471 ResultScanner results = table.getScanner(scan);
2472 MessageDigest digest = MessageDigest.getInstance("MD5");
2473 for (Result res : results) {
2474 digest.update(res.getRow());
2475 }
2476 results.close();
2477 return digest.toString();
2478 }
2479
2480
2481 public static final byte[][] ROWS = new byte[(int) Math.pow('z' - 'a' + 1, 3)][3];
2482 static {
2483 int i = 0;
2484 for (byte b1 = 'a'; b1 <= 'z'; b1++) {
2485 for (byte b2 = 'a'; b2 <= 'z'; b2++) {
2486 for (byte b3 = 'a'; b3 <= 'z'; b3++) {
2487 ROWS[i][0] = b1;
2488 ROWS[i][1] = b2;
2489 ROWS[i][2] = b3;
2490 i++;
2491 }
2492 }
2493 }
2494 }
2495
2496 public static final byte[][] KEYS = {
2497 HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes("bbb"),
2498 Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
2499 Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
2500 Bytes.toBytes("iii"), Bytes.toBytes("jjj"), Bytes.toBytes("kkk"),
2501 Bytes.toBytes("lll"), Bytes.toBytes("mmm"), Bytes.toBytes("nnn"),
2502 Bytes.toBytes("ooo"), Bytes.toBytes("ppp"), Bytes.toBytes("qqq"),
2503 Bytes.toBytes("rrr"), Bytes.toBytes("sss"), Bytes.toBytes("ttt"),
2504 Bytes.toBytes("uuu"), Bytes.toBytes("vvv"), Bytes.toBytes("www"),
2505 Bytes.toBytes("xxx"), Bytes.toBytes("yyy")
2506 };
2507
2508 public static final byte[][] KEYS_FOR_HBA_CREATE_TABLE = {
2509 Bytes.toBytes("bbb"),
2510 Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
2511 Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
2512 Bytes.toBytes("iii"), Bytes.toBytes("jjj"), Bytes.toBytes("kkk"),
2513 Bytes.toBytes("lll"), Bytes.toBytes("mmm"), Bytes.toBytes("nnn"),
2514 Bytes.toBytes("ooo"), Bytes.toBytes("ppp"), Bytes.toBytes("qqq"),
2515 Bytes.toBytes("rrr"), Bytes.toBytes("sss"), Bytes.toBytes("ttt"),
2516 Bytes.toBytes("uuu"), Bytes.toBytes("vvv"), Bytes.toBytes("www"),
2517 Bytes.toBytes("xxx"), Bytes.toBytes("yyy"), Bytes.toBytes("zzz")
2518 };
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530 public List<HRegionInfo> createMultiRegionsInMeta(final Configuration conf,
2531 final HTableDescriptor htd, byte [][] startKeys)
2532 throws IOException {
2533 Table meta = new HTable(conf, TableName.META_TABLE_NAME);
2534 Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR);
2535 List<HRegionInfo> newRegions = new ArrayList<HRegionInfo>(startKeys.length);
2536
2537 for (int i = 0; i < startKeys.length; i++) {
2538 int j = (i + 1) % startKeys.length;
2539 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKeys[i],
2540 startKeys[j]);
2541 MetaTableAccessor.addRegionToMeta(meta, hri);
2542 newRegions.add(hri);
2543 }
2544
2545 meta.close();
2546 return newRegions;
2547 }
2548
2549
2550
2551
2552
2553
2554 public List<byte[]> getMetaTableRows() throws IOException {
2555
2556 Table t = new HTable(new Configuration(this.conf), TableName.META_TABLE_NAME);
2557 List<byte[]> rows = new ArrayList<byte[]>();
2558 ResultScanner s = t.getScanner(new Scan());
2559 for (Result result : s) {
2560 LOG.info("getMetaTableRows: row -> " +
2561 Bytes.toStringBinary(result.getRow()));
2562 rows.add(result.getRow());
2563 }
2564 s.close();
2565 t.close();
2566 return rows;
2567 }
2568
2569
2570
2571
2572
2573
2574 public List<byte[]> getMetaTableRows(TableName tableName) throws IOException {
2575
2576 Table t = new HTable(new Configuration(this.conf), TableName.META_TABLE_NAME);
2577 List<byte[]> rows = new ArrayList<byte[]>();
2578 ResultScanner s = t.getScanner(new Scan());
2579 for (Result result : s) {
2580 HRegionInfo info = HRegionInfo.getHRegionInfo(result);
2581 if (info == null) {
2582 LOG.error("No region info for row " + Bytes.toString(result.getRow()));
2583
2584 continue;
2585 }
2586
2587 if (info.getTable().equals(tableName)) {
2588 LOG.info("getMetaTableRows: row -> " +
2589 Bytes.toStringBinary(result.getRow()) + info);
2590 rows.add(result.getRow());
2591 }
2592 }
2593 s.close();
2594 t.close();
2595 return rows;
2596 }
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609 public HRegionServer getRSForFirstRegionInTable(TableName tableName)
2610 throws IOException, InterruptedException {
2611 List<byte[]> metaRows = getMetaTableRows(tableName);
2612 if (metaRows == null || metaRows.isEmpty()) {
2613 return null;
2614 }
2615 LOG.debug("Found " + metaRows.size() + " rows for table " +
2616 tableName);
2617 byte [] firstrow = metaRows.get(0);
2618 LOG.debug("FirstRow=" + Bytes.toString(firstrow));
2619 long pause = getConfiguration().getLong(HConstants.HBASE_CLIENT_PAUSE,
2620 HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
2621 int numRetries = getConfiguration().getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
2622 HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
2623 RetryCounter retrier = new RetryCounter(numRetries+1, (int)pause, TimeUnit.MICROSECONDS);
2624 while(retrier.shouldRetry()) {
2625 int index = getMiniHBaseCluster().getServerWith(firstrow);
2626 if (index != -1) {
2627 return getMiniHBaseCluster().getRegionServerThreads().get(index).getRegionServer();
2628 }
2629
2630 retrier.sleepUntilNextRetry();
2631 }
2632 return null;
2633 }
2634
2635
2636
2637
2638
2639
2640
2641 public MiniMRCluster startMiniMapReduceCluster() throws IOException {
2642 startMiniMapReduceCluster(2);
2643 return mrCluster;
2644 }
2645
2646
2647
2648
2649
2650 private void forceChangeTaskLogDir() {
2651 Field logDirField;
2652 try {
2653 logDirField = TaskLog.class.getDeclaredField("LOG_DIR");
2654 logDirField.setAccessible(true);
2655
2656 Field modifiersField = Field.class.getDeclaredField("modifiers");
2657 modifiersField.setAccessible(true);
2658 modifiersField.setInt(logDirField, logDirField.getModifiers() & ~Modifier.FINAL);
2659
2660 logDirField.set(null, new File(hadoopLogDir, "userlogs"));
2661 } catch (SecurityException e) {
2662 throw new RuntimeException(e);
2663 } catch (NoSuchFieldException e) {
2664
2665 throw new RuntimeException(e);
2666 } catch (IllegalArgumentException e) {
2667 throw new RuntimeException(e);
2668 } catch (IllegalAccessException e) {
2669 throw new RuntimeException(e);
2670 }
2671 }
2672
2673
2674
2675
2676
2677
2678
2679 private void startMiniMapReduceCluster(final int servers) throws IOException {
2680 if (mrCluster != null) {
2681 throw new IllegalStateException("MiniMRCluster is already running");
2682 }
2683 LOG.info("Starting mini mapreduce cluster...");
2684 setupClusterTestDir();
2685 createDirsAndSetProperties();
2686
2687 forceChangeTaskLogDir();
2688
2689
2690
2691
2692 conf.setFloat("yarn.nodemanager.vmem-pmem-ratio", 8.0f);
2693
2694
2695
2696 conf.setBoolean("mapreduce.map.speculative", false);
2697 conf.setBoolean("mapreduce.reduce.speculative", false);
2698
2699
2700
2701 mrCluster = new MiniMRCluster(servers,
2702 FS_URI != null ? FS_URI : FileSystem.get(conf).getUri().toString(), 1,
2703 null, null, new JobConf(this.conf));
2704 JobConf jobConf = MapreduceTestingShim.getJobConf(mrCluster);
2705 if (jobConf == null) {
2706 jobConf = mrCluster.createJobConf();
2707 }
2708
2709 jobConf.set("mapreduce.cluster.local.dir",
2710 conf.get("mapreduce.cluster.local.dir"));
2711 LOG.info("Mini mapreduce cluster started");
2712
2713
2714
2715
2716 conf.set("mapreduce.jobtracker.address", jobConf.get("mapreduce.jobtracker.address"));
2717
2718 conf.set("mapreduce.framework.name", "yarn");
2719 conf.setBoolean("yarn.is.minicluster", true);
2720 String rmAddress = jobConf.get("yarn.resourcemanager.address");
2721 if (rmAddress != null) {
2722 conf.set("yarn.resourcemanager.address", rmAddress);
2723 }
2724 String historyAddress = jobConf.get("mapreduce.jobhistory.address");
2725 if (historyAddress != null) {
2726 conf.set("mapreduce.jobhistory.address", historyAddress);
2727 }
2728 String schedulerAddress =
2729 jobConf.get("yarn.resourcemanager.scheduler.address");
2730 if (schedulerAddress != null) {
2731 conf.set("yarn.resourcemanager.scheduler.address", schedulerAddress);
2732 }
2733 String mrJobHistoryWebappAddress =
2734 jobConf.get("mapreduce.jobhistory.webapp.address");
2735 if (mrJobHistoryWebappAddress != null) {
2736 conf.set("mapreduce.jobhistory.webapp.address", mrJobHistoryWebappAddress);
2737 }
2738 String yarnRMWebappAddress =
2739 jobConf.get("yarn.resourcemanager.webapp.address");
2740 if (yarnRMWebappAddress != null) {
2741 conf.set("yarn.resourcemanager.webapp.address", yarnRMWebappAddress);
2742 }
2743 }
2744
2745
2746
2747
2748 public void shutdownMiniMapReduceCluster() {
2749 if (mrCluster != null) {
2750 LOG.info("Stopping mini mapreduce cluster...");
2751 mrCluster.shutdown();
2752 mrCluster = null;
2753 LOG.info("Mini mapreduce cluster stopped");
2754 }
2755
2756 conf.set("mapreduce.jobtracker.address", "local");
2757 }
2758
2759
2760
2761
2762 public RegionServerServices createMockRegionServerService() throws IOException {
2763 return createMockRegionServerService((ServerName)null);
2764 }
2765
2766
2767
2768
2769
2770 public RegionServerServices createMockRegionServerService(RpcServerInterface rpc) throws IOException {
2771 final MockRegionServerServices rss = new MockRegionServerServices(getZooKeeperWatcher());
2772 rss.setFileSystem(getTestFileSystem());
2773 rss.setRpcServer(rpc);
2774 return rss;
2775 }
2776
2777
2778
2779
2780
2781 public RegionServerServices createMockRegionServerService(ServerName name) throws IOException {
2782 final MockRegionServerServices rss = new MockRegionServerServices(getZooKeeperWatcher(), name);
2783 rss.setFileSystem(getTestFileSystem());
2784 return rss;
2785 }
2786
2787
2788
2789
2790
2791
2792 public void enableDebug(Class<?> clazz) {
2793 Log l = LogFactory.getLog(clazz);
2794 if (l instanceof Log4JLogger) {
2795 ((Log4JLogger) l).getLogger().setLevel(org.apache.log4j.Level.DEBUG);
2796 } else if (l instanceof Jdk14Logger) {
2797 ((Jdk14Logger) l).getLogger().setLevel(java.util.logging.Level.ALL);
2798 }
2799 }
2800
2801
2802
2803
2804
2805 public void expireMasterSession() throws Exception {
2806 HMaster master = getMiniHBaseCluster().getMaster();
2807 expireSession(master.getZooKeeper(), false);
2808 }
2809
2810
2811
2812
2813
2814
2815 public void expireRegionServerSession(int index) throws Exception {
2816 HRegionServer rs = getMiniHBaseCluster().getRegionServer(index);
2817 expireSession(rs.getZooKeeper(), false);
2818 decrementMinRegionServerCount();
2819 }
2820
2821 private void decrementMinRegionServerCount() {
2822
2823
2824 decrementMinRegionServerCount(getConfiguration());
2825
2826
2827 for (MasterThread master : getHBaseCluster().getMasterThreads()) {
2828 decrementMinRegionServerCount(master.getMaster().getConfiguration());
2829 }
2830 }
2831
2832 private void decrementMinRegionServerCount(Configuration conf) {
2833 int currentCount = conf.getInt(
2834 ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1);
2835 if (currentCount != -1) {
2836 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
2837 Math.max(currentCount - 1, 1));
2838 }
2839 }
2840
2841 public void expireSession(ZooKeeperWatcher nodeZK) throws Exception {
2842 expireSession(nodeZK, false);
2843 }
2844
2845 @Deprecated
2846 public void expireSession(ZooKeeperWatcher nodeZK, Server server)
2847 throws Exception {
2848 expireSession(nodeZK, false);
2849 }
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862 public void expireSession(ZooKeeperWatcher nodeZK, boolean checkStatus)
2863 throws Exception {
2864 Configuration c = new Configuration(this.conf);
2865 String quorumServers = ZKConfig.getZKQuorumServersString(c);
2866 ZooKeeper zk = nodeZK.getRecoverableZooKeeper().getZooKeeper();
2867 byte[] password = zk.getSessionPasswd();
2868 long sessionID = zk.getSessionId();
2869
2870
2871
2872
2873
2874
2875
2876
2877 ZooKeeper monitor = new ZooKeeper(quorumServers,
2878 1000, new org.apache.zookeeper.Watcher(){
2879 @Override
2880 public void process(WatchedEvent watchedEvent) {
2881 LOG.info("Monitor ZKW received event="+watchedEvent);
2882 }
2883 } , sessionID, password);
2884
2885
2886 ZooKeeper newZK = new ZooKeeper(quorumServers,
2887 1000, EmptyWatcher.instance, sessionID, password);
2888
2889
2890
2891 long start = System.currentTimeMillis();
2892 while (newZK.getState() != States.CONNECTED
2893 && System.currentTimeMillis() - start < 1000) {
2894 Thread.sleep(1);
2895 }
2896 newZK.close();
2897 LOG.info("ZK Closed Session 0x" + Long.toHexString(sessionID));
2898
2899
2900 monitor.close();
2901
2902 if (checkStatus) {
2903 new HTable(new Configuration(conf), TableName.META_TABLE_NAME).close();
2904 }
2905 }
2906
2907
2908
2909
2910
2911
2912
2913 public MiniHBaseCluster getHBaseCluster() {
2914 return getMiniHBaseCluster();
2915 }
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925 public HBaseCluster getHBaseClusterInterface() {
2926
2927
2928 return hbaseCluster;
2929 }
2930
2931
2932
2933
2934
2935
2936
2937 public Connection getConnection() throws IOException {
2938 if (this.connection == null) {
2939 this.connection = ConnectionFactory.createConnection(this.conf);
2940 }
2941 return this.connection;
2942 }
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953 public synchronized HBaseAdmin getHBaseAdmin()
2954 throws IOException {
2955 if (hbaseAdmin == null){
2956 this.hbaseAdmin = new HBaseAdminForTests(getConnection());
2957 }
2958 return hbaseAdmin;
2959 }
2960
2961 private HBaseAdminForTests hbaseAdmin = null;
2962 private static class HBaseAdminForTests extends HBaseAdmin {
2963 public HBaseAdminForTests(Connection connection) throws MasterNotRunningException,
2964 ZooKeeperConnectionException, IOException {
2965 super(connection);
2966 }
2967
2968 @Override
2969 public synchronized void close() throws IOException {
2970 LOG.warn("close() called on HBaseAdmin instance returned from " +
2971 "HBaseTestingUtility.getHBaseAdmin()");
2972 }
2973
2974 private synchronized void close0() throws IOException {
2975 super.close();
2976 }
2977 }
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988 public synchronized ZooKeeperWatcher getZooKeeperWatcher()
2989 throws IOException {
2990 if (zooKeeperWatcher == null) {
2991 zooKeeperWatcher = new ZooKeeperWatcher(conf, "testing utility",
2992 new Abortable() {
2993 @Override public void abort(String why, Throwable e) {
2994 throw new RuntimeException("Unexpected abort in HBaseTestingUtility:"+why, e);
2995 }
2996 @Override public boolean isAborted() {return false;}
2997 });
2998 }
2999 return zooKeeperWatcher;
3000 }
3001 private ZooKeeperWatcher zooKeeperWatcher;
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011 public void closeRegion(String regionName) throws IOException {
3012 closeRegion(Bytes.toBytes(regionName));
3013 }
3014
3015
3016
3017
3018
3019
3020
3021 public void closeRegion(byte[] regionName) throws IOException {
3022 getHBaseAdmin().closeRegion(regionName, null);
3023 }
3024
3025
3026
3027
3028
3029
3030
3031
3032 public void closeRegionByRow(String row, RegionLocator table) throws IOException {
3033 closeRegionByRow(Bytes.toBytes(row), table);
3034 }
3035
3036
3037
3038
3039
3040
3041
3042
3043 public void closeRegionByRow(byte[] row, RegionLocator table) throws IOException {
3044 HRegionLocation hrl = table.getRegionLocation(row);
3045 closeRegion(hrl.getRegionInfo().getRegionName());
3046 }
3047
3048
3049
3050
3051
3052
3053
3054
3055 public HRegion getSplittableRegion(TableName tableName, int maxAttempts) {
3056 List<HRegion> regions = getHBaseCluster().getRegions(tableName);
3057 int regCount = regions.size();
3058 Set<Integer> attempted = new HashSet<Integer>();
3059 int idx;
3060 int attempts = 0;
3061 do {
3062 regions = getHBaseCluster().getRegions(tableName);
3063 if (regCount != regions.size()) {
3064
3065 attempted.clear();
3066 }
3067 regCount = regions.size();
3068
3069
3070 if (regCount > 0) {
3071 idx = random.nextInt(regCount);
3072
3073 if (attempted.contains(idx))
3074 continue;
3075 try {
3076 regions.get(idx).checkSplit();
3077 return regions.get(idx);
3078 } catch (Exception ex) {
3079 LOG.warn("Caught exception", ex);
3080 attempted.add(idx);
3081 }
3082 }
3083 attempts++;
3084 } while (maxAttempts == -1 || attempts < maxAttempts);
3085 return null;
3086 }
3087
3088 public MiniZooKeeperCluster getZkCluster() {
3089 return zkCluster;
3090 }
3091
3092 public void setZkCluster(MiniZooKeeperCluster zkCluster) {
3093 this.passedZkCluster = true;
3094 this.zkCluster = zkCluster;
3095 conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkCluster.getClientPort());
3096 }
3097
3098 public MiniDFSCluster getDFSCluster() {
3099 return dfsCluster;
3100 }
3101
3102 public void setDFSCluster(MiniDFSCluster cluster) throws IllegalStateException, IOException {
3103 setDFSCluster(cluster, true);
3104 }
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114 public void setDFSCluster(MiniDFSCluster cluster, boolean requireDown)
3115 throws IllegalStateException, IOException {
3116 if (dfsCluster != null && requireDown && dfsCluster.isClusterUp()) {
3117 throw new IllegalStateException("DFSCluster is already running! Shut it down first.");
3118 }
3119 this.dfsCluster = cluster;
3120 this.setFs();
3121 }
3122
3123 public FileSystem getTestFileSystem() throws IOException {
3124 return HFileSystem.get(conf);
3125 }
3126
3127
3128
3129
3130
3131
3132
3133
3134 public void waitTableAvailable(TableName table)
3135 throws InterruptedException, IOException {
3136 waitTableAvailable(table.getName(), 30000);
3137 }
3138
3139 public void waitTableAvailable(TableName table, long timeoutMillis)
3140 throws InterruptedException, IOException {
3141 waitFor(timeoutMillis, predicateTableAvailable(table));
3142 }
3143
3144 public String explainTableAvailability(TableName tableName) throws IOException {
3145 String msg = explainTableState(tableName) + ",";
3146 if (getHBaseCluster().getMaster().isAlive()) {
3147 Map<HRegionInfo, ServerName> assignments =
3148 getHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
3149 .getRegionAssignments();
3150 final List<Pair<HRegionInfo, ServerName>> metaLocations =
3151 MetaTableAccessor
3152 .getTableRegionsAndLocations(getZooKeeperWatcher(), connection, tableName);
3153 for (Pair<HRegionInfo, ServerName> metaLocation : metaLocations) {
3154 HRegionInfo hri = metaLocation.getFirst();
3155 ServerName sn = metaLocation.getSecond();
3156 if (!assignments.containsKey(hri)) {
3157 msg += ", region " + hri
3158 + " not assigned, but found in meta, it expected to be on " + sn;
3159
3160 } else if (sn == null) {
3161 msg += ", region " + hri
3162 + " assigned, but has no server in meta";
3163 } else if (!sn.equals(assignments.get(hri))) {
3164 msg += ", region " + hri
3165 + " assigned, but has different servers in meta and AM ( " +
3166 sn + " <> " + assignments.get(hri);
3167 }
3168 }
3169 }
3170 return msg;
3171 }
3172
3173 public String explainTableState(TableName tableName) throws IOException {
3174 try {
3175 if (getHBaseAdmin().isTableEnabled(tableName))
3176 return "table enabled in zk";
3177 else if (getHBaseAdmin().isTableDisabled(tableName))
3178 return "table disabled in zk";
3179 else
3180 return "table in uknown state";
3181 } catch (TableNotFoundException e) {
3182 return "table not exists";
3183 }
3184 }
3185
3186
3187
3188
3189
3190
3191
3192
3193 public void waitTableAvailable(byte[] table, long timeoutMillis)
3194 throws InterruptedException, IOException {
3195 waitTableAvailable(getHBaseAdmin(), table, timeoutMillis);
3196 }
3197
3198 public void waitTableAvailable(Admin admin, byte[] table, long timeoutMillis)
3199 throws InterruptedException, IOException {
3200 long startWait = System.currentTimeMillis();
3201 while (!admin.isTableAvailable(TableName.valueOf(table))) {
3202 assertTrue("Timed out waiting for table to become available " +
3203 Bytes.toStringBinary(table),
3204 System.currentTimeMillis() - startWait < timeoutMillis);
3205 Thread.sleep(200);
3206 }
3207 }
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218 public void waitTableEnabled(TableName table)
3219 throws InterruptedException, IOException {
3220 waitTableEnabled(table, 30000);
3221 }
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232 public void waitTableEnabled(byte[] table, long timeoutMillis)
3233 throws InterruptedException, IOException {
3234 waitTableEnabled(TableName.valueOf(table), timeoutMillis);
3235 }
3236
3237 public void waitTableEnabled(TableName table, long timeoutMillis)
3238 throws IOException {
3239 waitFor(timeoutMillis, predicateTableEnabled(table));
3240 }
3241
3242
3243
3244
3245
3246
3247
3248
3249 public void waitTableDisabled(byte[] table)
3250 throws InterruptedException, IOException {
3251 waitTableDisabled(getHBaseAdmin(), table, 30000);
3252 }
3253
3254 public void waitTableDisabled(Admin admin, byte[] table)
3255 throws InterruptedException, IOException {
3256 waitTableDisabled(admin, table, 30000);
3257 }
3258
3259
3260
3261
3262
3263
3264
3265
3266 public void waitTableDisabled(byte[] table, long timeoutMillis)
3267 throws InterruptedException, IOException {
3268 waitTableDisabled(getHBaseAdmin(), table, timeoutMillis);
3269 }
3270
3271 public void waitTableDisabled(Admin admin, byte[] table, long timeoutMillis)
3272 throws InterruptedException, IOException {
3273 TableName tableName = TableName.valueOf(table);
3274 long startWait = System.currentTimeMillis();
3275 while (!admin.isTableDisabled(tableName)) {
3276 assertTrue("Timed out waiting for table to become disabled " +
3277 Bytes.toStringBinary(table),
3278 System.currentTimeMillis() - startWait < timeoutMillis);
3279 Thread.sleep(200);
3280 }
3281 }
3282
3283
3284
3285
3286
3287
3288
3289
3290 public boolean ensureSomeRegionServersAvailable(final int num)
3291 throws IOException {
3292 boolean startedServer = false;
3293 MiniHBaseCluster hbaseCluster = getMiniHBaseCluster();
3294 for (int i=hbaseCluster.getLiveRegionServerThreads().size(); i<num; ++i) {
3295 LOG.info("Started new server=" + hbaseCluster.startRegionServer());
3296 startedServer = true;
3297 }
3298
3299 return startedServer;
3300 }
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311 public boolean ensureSomeNonStoppedRegionServersAvailable(final int num)
3312 throws IOException {
3313 boolean startedServer = ensureSomeRegionServersAvailable(num);
3314
3315 int nonStoppedServers = 0;
3316 for (JVMClusterUtil.RegionServerThread rst :
3317 getMiniHBaseCluster().getRegionServerThreads()) {
3318
3319 HRegionServer hrs = rst.getRegionServer();
3320 if (hrs.isStopping() || hrs.isStopped()) {
3321 LOG.info("A region server is stopped or stopping:"+hrs);
3322 } else {
3323 nonStoppedServers++;
3324 }
3325 }
3326 for (int i=nonStoppedServers; i<num; ++i) {
3327 LOG.info("Started new server=" + getMiniHBaseCluster().startRegionServer());
3328 startedServer = true;
3329 }
3330 return startedServer;
3331 }
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343 public static User getDifferentUser(final Configuration c,
3344 final String differentiatingSuffix)
3345 throws IOException {
3346 FileSystem currentfs = FileSystem.get(c);
3347 if (!(currentfs instanceof DistributedFileSystem) || User.isHBaseSecurityEnabled(c)) {
3348 return User.getCurrent();
3349 }
3350
3351
3352 String username = User.getCurrent().getName() +
3353 differentiatingSuffix;
3354 User user = User.createUserForTesting(c, username,
3355 new String[]{"supergroup"});
3356 return user;
3357 }
3358
3359 public static NavigableSet<String> getAllOnlineRegions(MiniHBaseCluster cluster)
3360 throws IOException {
3361 NavigableSet<String> online = new TreeSet<String>();
3362 for (RegionServerThread rst : cluster.getLiveRegionServerThreads()) {
3363 try {
3364 for (HRegionInfo region :
3365 ProtobufUtil.getOnlineRegions(rst.getRegionServer().getRSRpcServices())) {
3366 online.add(region.getRegionNameAsString());
3367 }
3368 } catch (RegionServerStoppedException e) {
3369
3370 }
3371 }
3372 for (MasterThread mt : cluster.getLiveMasterThreads()) {
3373 try {
3374 for (HRegionInfo region :
3375 ProtobufUtil.getOnlineRegions(mt.getMaster().getRSRpcServices())) {
3376 online.add(region.getRegionNameAsString());
3377 }
3378 } catch (RegionServerStoppedException e) {
3379
3380 } catch (ServerNotRunningYetException e) {
3381
3382 }
3383 }
3384 return online;
3385 }
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400 public static void setMaxRecoveryErrorCount(final OutputStream stream,
3401 final int max) {
3402 try {
3403 Class<?> [] clazzes = DFSClient.class.getDeclaredClasses();
3404 for (Class<?> clazz: clazzes) {
3405 String className = clazz.getSimpleName();
3406 if (className.equals("DFSOutputStream")) {
3407 if (clazz.isInstance(stream)) {
3408 Field maxRecoveryErrorCountField =
3409 stream.getClass().getDeclaredField("maxRecoveryErrorCount");
3410 maxRecoveryErrorCountField.setAccessible(true);
3411 maxRecoveryErrorCountField.setInt(stream, max);
3412 break;
3413 }
3414 }
3415 }
3416 } catch (Exception e) {
3417 LOG.info("Could not set max recovery field", e);
3418 }
3419 }
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430 public void waitUntilAllRegionsAssigned(final TableName tableName) throws IOException {
3431 waitUntilAllRegionsAssigned(
3432 tableName,
3433 this.conf.getLong("hbase.client.sync.wait.timeout.msec", 60000));
3434 }
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445 public void waitUntilAllRegionsAssigned(final TableName tableName, final long timeout)
3446 throws IOException {
3447 final Table meta = new HTable(getConfiguration(), TableName.META_TABLE_NAME);
3448 try {
3449 waitFor(timeout, 200, true, new Predicate<IOException>() {
3450 @Override
3451 public boolean evaluate() throws IOException {
3452 boolean allRegionsAssigned = true;
3453 Scan scan = new Scan();
3454 scan.addFamily(HConstants.CATALOG_FAMILY);
3455 ResultScanner s = meta.getScanner(scan);
3456 try {
3457 Result r;
3458 while ((r = s.next()) != null) {
3459 byte[] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
3460 HRegionInfo info = HRegionInfo.parseFromOrNull(b);
3461 if (info != null && info.getTable().equals(tableName)) {
3462 b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
3463 allRegionsAssigned &= (b != null);
3464 }
3465 }
3466 } finally {
3467 s.close();
3468 }
3469 return allRegionsAssigned;
3470 }
3471 });
3472 } finally {
3473 meta.close();
3474 }
3475
3476
3477 if (!getHBaseClusterInterface().isDistributedCluster()) {
3478
3479
3480 HMaster master = getHBaseCluster().getMaster();
3481 final RegionStates states = master.getAssignmentManager().getRegionStates();
3482 waitFor(timeout, 200, new ExplainingPredicate<IOException>() {
3483 @Override
3484 public String explainFailure() throws IOException {
3485 return explainTableAvailability(tableName);
3486 }
3487
3488 @Override
3489 public boolean evaluate() throws IOException {
3490 List<HRegionInfo> hris = states.getRegionsOfTable(tableName);
3491 return hris != null && !hris.isEmpty();
3492 }
3493 });
3494 }
3495 }
3496
3497
3498
3499
3500
3501 public static List<Cell> getFromStoreFile(HStore store,
3502 Get get) throws IOException {
3503 Scan scan = new Scan(get);
3504 InternalScanner scanner = (InternalScanner) store.getScanner(scan,
3505 scan.getFamilyMap().get(store.getFamily().getName()),
3506
3507
3508 0);
3509
3510 List<Cell> result = new ArrayList<Cell>();
3511 scanner.next(result);
3512 if (!result.isEmpty()) {
3513
3514 Cell kv = result.get(0);
3515 if (!CellUtil.matchingRow(kv, get.getRow())) {
3516 result.clear();
3517 }
3518 }
3519 scanner.close();
3520 return result;
3521 }
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531 public byte[][] getRegionSplitStartKeys(byte[] startKey, byte[] endKey, int numRegions){
3532 assertTrue(numRegions>3);
3533 byte [][] tmpSplitKeys = Bytes.split(startKey, endKey, numRegions - 3);
3534 byte [][] result = new byte[tmpSplitKeys.length+1][];
3535 System.arraycopy(tmpSplitKeys, 0, result, 1, tmpSplitKeys.length);
3536 result[0] = HConstants.EMPTY_BYTE_ARRAY;
3537 return result;
3538 }
3539
3540
3541
3542
3543
3544 public static List<Cell> getFromStoreFile(HStore store,
3545 byte [] row,
3546 NavigableSet<byte[]> columns
3547 ) throws IOException {
3548 Get get = new Get(row);
3549 Map<byte[], NavigableSet<byte[]>> s = get.getFamilyMap();
3550 s.put(store.getFamily().getName(), columns);
3551
3552 return getFromStoreFile(store,get);
3553 }
3554
3555
3556
3557
3558
3559 public static ZooKeeperWatcher getZooKeeperWatcher(
3560 HBaseTestingUtility TEST_UTIL) throws ZooKeeperConnectionException,
3561 IOException {
3562 ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
3563 "unittest", new Abortable() {
3564 boolean aborted = false;
3565
3566 @Override
3567 public void abort(String why, Throwable e) {
3568 aborted = true;
3569 throw new RuntimeException("Fatal ZK error, why=" + why, e);
3570 }
3571
3572 @Override
3573 public boolean isAborted() {
3574 return aborted;
3575 }
3576 });
3577 return zkw;
3578 }
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591 public static ZooKeeperWatcher createAndForceNodeToOpenedState(
3592 HBaseTestingUtility TEST_UTIL, Region region,
3593 ServerName serverName) throws ZooKeeperConnectionException,
3594 IOException, KeeperException, NodeExistsException {
3595 return createAndForceNodeToOpenedState(TEST_UTIL, (HRegion)region, serverName);
3596 }
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609 public static ZooKeeperWatcher createAndForceNodeToOpenedState(
3610 HBaseTestingUtility TEST_UTIL, HRegion region,
3611 ServerName serverName) throws ZooKeeperConnectionException,
3612 IOException, KeeperException, NodeExistsException {
3613 ZooKeeperWatcher zkw = getZooKeeperWatcher(TEST_UTIL);
3614 ZKAssign.createNodeOffline(zkw, region.getRegionInfo(), serverName);
3615 int version = ZKAssign.transitionNodeOpening(zkw, region
3616 .getRegionInfo(), serverName);
3617 ZKAssign.transitionNodeOpened(zkw, region.getRegionInfo(), serverName,
3618 version);
3619 return zkw;
3620 }
3621
3622 public static void assertKVListsEqual(String additionalMsg,
3623 final List<? extends Cell> expected,
3624 final List<? extends Cell> actual) {
3625 final int eLen = expected.size();
3626 final int aLen = actual.size();
3627 final int minLen = Math.min(eLen, aLen);
3628
3629 int i;
3630 for (i = 0; i < minLen
3631 && KeyValue.COMPARATOR.compare(expected.get(i), actual.get(i)) == 0;
3632 ++i) {}
3633
3634 if (additionalMsg == null) {
3635 additionalMsg = "";
3636 }
3637 if (!additionalMsg.isEmpty()) {
3638 additionalMsg = ". " + additionalMsg;
3639 }
3640
3641 if (eLen != aLen || i != minLen) {
3642 throw new AssertionError(
3643 "Expected and actual KV arrays differ at position " + i + ": " +
3644 safeGetAsStr(expected, i) + " (length " + eLen +") vs. " +
3645 safeGetAsStr(actual, i) + " (length " + aLen + ")" + additionalMsg);
3646 }
3647 }
3648
3649 public static <T> String safeGetAsStr(List<T> lst, int i) {
3650 if (0 <= i && i < lst.size()) {
3651 return lst.get(i).toString();
3652 } else {
3653 return "<out_of_range>";
3654 }
3655 }
3656
3657 public String getClusterKey() {
3658 return conf.get(HConstants.ZOOKEEPER_QUORUM) + ":"
3659 + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT) + ":"
3660 + conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
3661 HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
3662 }
3663
3664
3665 public HTable createRandomTable(String tableName,
3666 final Collection<String> families,
3667 final int maxVersions,
3668 final int numColsPerRow,
3669 final int numFlushes,
3670 final int numRegions,
3671 final int numRowsPerFlush)
3672 throws IOException, InterruptedException {
3673
3674 LOG.info("\n\nCreating random table " + tableName + " with " + numRegions +
3675 " regions, " + numFlushes + " storefiles per region, " +
3676 numRowsPerFlush + " rows per flush, maxVersions=" + maxVersions +
3677 "\n");
3678
3679 final Random rand = new Random(tableName.hashCode() * 17L + 12938197137L);
3680 final int numCF = families.size();
3681 final byte[][] cfBytes = new byte[numCF][];
3682 {
3683 int cfIndex = 0;
3684 for (String cf : families) {
3685 cfBytes[cfIndex++] = Bytes.toBytes(cf);
3686 }
3687 }
3688
3689 final int actualStartKey = 0;
3690 final int actualEndKey = Integer.MAX_VALUE;
3691 final int keysPerRegion = (actualEndKey - actualStartKey) / numRegions;
3692 final int splitStartKey = actualStartKey + keysPerRegion;
3693 final int splitEndKey = actualEndKey - keysPerRegion;
3694 final String keyFormat = "%08x";
3695 final HTable table = createTable(tableName, cfBytes,
3696 maxVersions,
3697 Bytes.toBytes(String.format(keyFormat, splitStartKey)),
3698 Bytes.toBytes(String.format(keyFormat, splitEndKey)),
3699 numRegions);
3700
3701 if (hbaseCluster != null) {
3702 getMiniHBaseCluster().flushcache(TableName.META_TABLE_NAME);
3703 }
3704
3705 for (int iFlush = 0; iFlush < numFlushes; ++iFlush) {
3706 for (int iRow = 0; iRow < numRowsPerFlush; ++iRow) {
3707 final byte[] row = Bytes.toBytes(String.format(keyFormat,
3708 actualStartKey + rand.nextInt(actualEndKey - actualStartKey)));
3709
3710 Put put = new Put(row);
3711 Delete del = new Delete(row);
3712 for (int iCol = 0; iCol < numColsPerRow; ++iCol) {
3713 final byte[] cf = cfBytes[rand.nextInt(numCF)];
3714 final long ts = rand.nextInt();
3715 final byte[] qual = Bytes.toBytes("col" + iCol);
3716 if (rand.nextBoolean()) {
3717 final byte[] value = Bytes.toBytes("value_for_row_" + iRow +
3718 "_cf_" + Bytes.toStringBinary(cf) + "_col_" + iCol + "_ts_" +
3719 ts + "_random_" + rand.nextLong());
3720 put.add(cf, qual, ts, value);
3721 } else if (rand.nextDouble() < 0.8) {
3722 del.deleteColumn(cf, qual, ts);
3723 } else {
3724 del.deleteColumns(cf, qual, ts);
3725 }
3726 }
3727
3728 if (!put.isEmpty()) {
3729 table.put(put);
3730 }
3731
3732 if (!del.isEmpty()) {
3733 table.delete(del);
3734 }
3735 }
3736 LOG.info("Initiating flush #" + iFlush + " for table " + tableName);
3737 table.flushCommits();
3738 if (hbaseCluster != null) {
3739 getMiniHBaseCluster().flushcache(table.getName());
3740 }
3741 }
3742
3743 return table;
3744 }
3745
3746 private static final int MIN_RANDOM_PORT = 0xc000;
3747 private static final int MAX_RANDOM_PORT = 0xfffe;
3748 private static Random random = new Random();
3749
3750
3751
3752
3753
3754 public static int randomPort() {
3755 return MIN_RANDOM_PORT
3756 + random.nextInt(MAX_RANDOM_PORT - MIN_RANDOM_PORT);
3757 }
3758
3759
3760
3761
3762
3763 public static int randomFreePort() {
3764 int port = 0;
3765 do {
3766 port = randomPort();
3767 if (takenRandomPorts.contains(port)) {
3768 port = 0;
3769 continue;
3770 }
3771 takenRandomPorts.add(port);
3772
3773 try {
3774 ServerSocket sock = new ServerSocket(port);
3775 sock.close();
3776 } catch (IOException ex) {
3777 port = 0;
3778 }
3779 } while (port == 0);
3780 return port;
3781 }
3782
3783
3784 public static String randomMultiCastAddress() {
3785 return "226.1.1." + random.nextInt(254);
3786 }
3787
3788
3789
3790 public static void waitForHostPort(String host, int port)
3791 throws IOException {
3792 final int maxTimeMs = 10000;
3793 final int maxNumAttempts = maxTimeMs / HConstants.SOCKET_RETRY_WAIT_MS;
3794 IOException savedException = null;
3795 LOG.info("Waiting for server at " + host + ":" + port);
3796 for (int attempt = 0; attempt < maxNumAttempts; ++attempt) {
3797 try {
3798 Socket sock = new Socket(InetAddress.getByName(host), port);
3799 sock.close();
3800 savedException = null;
3801 LOG.info("Server at " + host + ":" + port + " is available");
3802 break;
3803 } catch (UnknownHostException e) {
3804 throw new IOException("Failed to look up " + host, e);
3805 } catch (IOException e) {
3806 savedException = e;
3807 }
3808 Threads.sleepWithoutInterrupt(HConstants.SOCKET_RETRY_WAIT_MS);
3809 }
3810
3811 if (savedException != null) {
3812 throw savedException;
3813 }
3814 }
3815
3816
3817
3818
3819
3820
3821 public static int createPreSplitLoadTestTable(Configuration conf,
3822 TableName tableName, byte[] columnFamily, Algorithm compression,
3823 DataBlockEncoding dataBlockEncoding) throws IOException {
3824 return createPreSplitLoadTestTable(conf, tableName,
3825 columnFamily, compression, dataBlockEncoding, DEFAULT_REGIONS_PER_SERVER, 1,
3826 Durability.USE_DEFAULT);
3827 }
3828
3829
3830
3831
3832
3833 public static int createPreSplitLoadTestTable(Configuration conf,
3834 TableName tableName, byte[] columnFamily, Algorithm compression,
3835 DataBlockEncoding dataBlockEncoding, int numRegionsPerServer, int regionReplication,
3836 Durability durability)
3837 throws IOException {
3838 HTableDescriptor desc = new HTableDescriptor(tableName);
3839 desc.setDurability(durability);
3840 desc.setRegionReplication(regionReplication);
3841 HColumnDescriptor hcd = new HColumnDescriptor(columnFamily);
3842 hcd.setDataBlockEncoding(dataBlockEncoding);
3843 hcd.setCompressionType(compression);
3844 return createPreSplitLoadTestTable(conf, desc, hcd, numRegionsPerServer);
3845 }
3846
3847
3848
3849
3850
3851
3852 public static int createPreSplitLoadTestTable(Configuration conf,
3853 TableName tableName, byte[][] columnFamilies, Algorithm compression,
3854 DataBlockEncoding dataBlockEncoding, int numRegionsPerServer, int regionReplication,
3855 Durability durability)
3856 throws IOException {
3857 HTableDescriptor desc = new HTableDescriptor(tableName);
3858 desc.setDurability(durability);
3859 desc.setRegionReplication(regionReplication);
3860 HColumnDescriptor[] hcds = new HColumnDescriptor[columnFamilies.length];
3861 for (int i = 0; i < columnFamilies.length; i++) {
3862 HColumnDescriptor hcd = new HColumnDescriptor(columnFamilies[i]);
3863 hcd.setDataBlockEncoding(dataBlockEncoding);
3864 hcd.setCompressionType(compression);
3865 hcds[i] = hcd;
3866 }
3867 return createPreSplitLoadTestTable(conf, desc, hcds, numRegionsPerServer);
3868 }
3869
3870
3871
3872
3873
3874
3875 public static int createPreSplitLoadTestTable(Configuration conf,
3876 HTableDescriptor desc, HColumnDescriptor hcd) throws IOException {
3877 return createPreSplitLoadTestTable(conf, desc, hcd, DEFAULT_REGIONS_PER_SERVER);
3878 }
3879
3880
3881
3882
3883
3884
3885 public static int createPreSplitLoadTestTable(Configuration conf,
3886 HTableDescriptor desc, HColumnDescriptor hcd, int numRegionsPerServer) throws IOException {
3887 return createPreSplitLoadTestTable(conf, desc, new HColumnDescriptor[] {hcd},
3888 numRegionsPerServer);
3889 }
3890
3891
3892
3893
3894
3895
3896 public static int createPreSplitLoadTestTable(Configuration conf,
3897 HTableDescriptor desc, HColumnDescriptor[] hcds, int numRegionsPerServer) throws IOException {
3898 for (HColumnDescriptor hcd : hcds) {
3899 if (!desc.hasFamily(hcd.getName())) {
3900 desc.addFamily(hcd);
3901 }
3902 }
3903
3904 int totalNumberOfRegions = 0;
3905 Connection unmanagedConnection = ConnectionFactory.createConnection(conf);
3906 Admin admin = unmanagedConnection.getAdmin();
3907
3908 try {
3909
3910
3911
3912 int numberOfServers = admin.getClusterStatus().getServers().size();
3913 if (numberOfServers == 0) {
3914 throw new IllegalStateException("No live regionservers");
3915 }
3916
3917 totalNumberOfRegions = numberOfServers * numRegionsPerServer;
3918 LOG.info("Number of live regionservers: " + numberOfServers + ", " +
3919 "pre-splitting table into " + totalNumberOfRegions + " regions " +
3920 "(regions per server: " + numRegionsPerServer + ")");
3921
3922 byte[][] splits = new RegionSplitter.HexStringSplit().split(
3923 totalNumberOfRegions);
3924
3925 admin.createTable(desc, splits);
3926 } catch (MasterNotRunningException e) {
3927 LOG.error("Master not running", e);
3928 throw new IOException(e);
3929 } catch (TableExistsException e) {
3930 LOG.warn("Table " + desc.getTableName() +
3931 " already exists, continuing");
3932 } finally {
3933 admin.close();
3934 unmanagedConnection.close();
3935 }
3936 return totalNumberOfRegions;
3937 }
3938
3939 public static int getMetaRSPort(Configuration conf) throws IOException {
3940 try (Connection c = ConnectionFactory.createConnection();
3941 RegionLocator locator = c.getRegionLocator(TableName.META_TABLE_NAME)) {
3942 return locator.getRegionLocation(Bytes.toBytes("")).getPort();
3943 }
3944 }
3945
3946
3947
3948
3949
3950
3951
3952 public void assertRegionOnServer(
3953 final HRegionInfo hri, final ServerName server,
3954 final long timeout) throws IOException, InterruptedException {
3955 long timeoutTime = System.currentTimeMillis() + timeout;
3956 while (true) {
3957 List<HRegionInfo> regions = getHBaseAdmin().getOnlineRegions(server);
3958 if (regions.contains(hri)) return;
3959 long now = System.currentTimeMillis();
3960 if (now > timeoutTime) break;
3961 Thread.sleep(10);
3962 }
3963 fail("Could not find region " + hri.getRegionNameAsString()
3964 + " on server " + server);
3965 }
3966
3967
3968
3969
3970
3971 public void assertRegionOnlyOnServer(
3972 final HRegionInfo hri, final ServerName server,
3973 final long timeout) throws IOException, InterruptedException {
3974 long timeoutTime = System.currentTimeMillis() + timeout;
3975 while (true) {
3976 List<HRegionInfo> regions = getHBaseAdmin().getOnlineRegions(server);
3977 if (regions.contains(hri)) {
3978 List<JVMClusterUtil.RegionServerThread> rsThreads =
3979 getHBaseCluster().getLiveRegionServerThreads();
3980 for (JVMClusterUtil.RegionServerThread rsThread: rsThreads) {
3981 HRegionServer rs = rsThread.getRegionServer();
3982 if (server.equals(rs.getServerName())) {
3983 continue;
3984 }
3985 Collection<Region> hrs = rs.getOnlineRegionsLocalContext();
3986 for (Region r: hrs) {
3987 assertTrue("Region should not be double assigned",
3988 r.getRegionInfo().getRegionId() != hri.getRegionId());
3989 }
3990 }
3991 return;
3992 }
3993 long now = System.currentTimeMillis();
3994 if (now > timeoutTime) break;
3995 Thread.sleep(10);
3996 }
3997 fail("Could not find region " + hri.getRegionNameAsString()
3998 + " on server " + server);
3999 }
4000
4001 public HRegion createTestRegion(String tableName, HColumnDescriptor hcd)
4002 throws IOException {
4003 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
4004 htd.addFamily(hcd);
4005 HRegionInfo info =
4006 new HRegionInfo(TableName.valueOf(tableName), null, null, false);
4007 HRegion region =
4008 HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), htd);
4009 return region;
4010 }
4011
4012 public void setFileSystemURI(String fsURI) {
4013 FS_URI = fsURI;
4014 }
4015
4016
4017
4018
4019 public <E extends Exception> long waitFor(long timeout, Predicate<E> predicate)
4020 throws E {
4021 return Waiter.waitFor(this.conf, timeout, predicate);
4022 }
4023
4024
4025
4026
4027 public <E extends Exception> long waitFor(long timeout, long interval, Predicate<E> predicate)
4028 throws E {
4029 return Waiter.waitFor(this.conf, timeout, interval, predicate);
4030 }
4031
4032
4033
4034
4035 public <E extends Exception> long waitFor(long timeout, long interval,
4036 boolean failIfTimeout, Predicate<E> predicate) throws E {
4037 return Waiter.waitFor(this.conf, timeout, interval, failIfTimeout, predicate);
4038 }
4039
4040
4041
4042
4043 public ExplainingPredicate<IOException> predicateNoRegionsInTransition() {
4044 return new ExplainingPredicate<IOException>() {
4045 @Override
4046 public String explainFailure() throws IOException {
4047 final RegionStates regionStates = getMiniHBaseCluster().getMaster()
4048 .getAssignmentManager().getRegionStates();
4049 return "found in transition: " + regionStates.getRegionsInTransition().toString();
4050 }
4051
4052 @Override
4053 public boolean evaluate() throws IOException {
4054 HMaster master = getMiniHBaseCluster().getMaster();
4055 if (master == null) return false;
4056 AssignmentManager am = master.getAssignmentManager();
4057 if (am == null) return false;
4058 final RegionStates regionStates = am.getRegionStates();
4059 return !regionStates.isRegionsInTransition();
4060 }
4061 };
4062 }
4063
4064
4065
4066
4067 public Waiter.Predicate<IOException> predicateTableEnabled(final TableName tableName) {
4068 return new ExplainingPredicate<IOException>() {
4069 @Override
4070 public String explainFailure() throws IOException {
4071 return explainTableState(tableName);
4072 }
4073
4074 @Override
4075 public boolean evaluate() throws IOException {
4076 return getHBaseAdmin().tableExists(tableName) && getHBaseAdmin().isTableEnabled(tableName);
4077 }
4078 };
4079 }
4080
4081
4082
4083
4084 public Waiter.Predicate<IOException> predicateTableDisabled(final TableName tableName) {
4085 return new ExplainingPredicate<IOException>() {
4086 @Override
4087 public String explainFailure() throws IOException {
4088 return explainTableState(tableName);
4089 }
4090
4091 @Override
4092 public boolean evaluate() throws IOException {
4093 return getHBaseAdmin().isTableDisabled(tableName);
4094 }
4095 };
4096 }
4097
4098
4099
4100
4101 public Waiter.Predicate<IOException> predicateTableAvailable(final TableName tableName) {
4102 return new ExplainingPredicate<IOException>() {
4103 @Override
4104 public String explainFailure() throws IOException {
4105 return explainTableAvailability(tableName);
4106 }
4107
4108 @Override
4109 public boolean evaluate() throws IOException {
4110 boolean tableAvailable = getHBaseAdmin().isTableAvailable(tableName);
4111 if (tableAvailable) {
4112 try {
4113 Canary.sniff(getHBaseAdmin(), tableName);
4114 } catch (Exception e) {
4115 throw new IOException("Canary sniff failed for table " + tableName, e);
4116 }
4117 }
4118 return tableAvailable;
4119 }
4120 };
4121 }
4122
4123
4124
4125
4126
4127
4128 public void waitUntilNoRegionsInTransition(final long timeout) throws IOException {
4129 waitFor(timeout, predicateNoRegionsInTransition());
4130 }
4131
4132
4133
4134
4135
4136 public void waitUntilNoRegionsInTransition() throws IOException {
4137 waitUntilNoRegionsInTransition(15 * 60000);
4138 }
4139
4140
4141
4142
4143
4144
4145 public void waitLabelAvailable(long timeoutMillis, final String... labels) {
4146 final VisibilityLabelsCache labelsCache = VisibilityLabelsCache.get();
4147 waitFor(timeoutMillis, new Waiter.ExplainingPredicate<RuntimeException>() {
4148
4149 @Override
4150 public boolean evaluate() {
4151 for (String label : labels) {
4152 if (labelsCache.getLabelOrdinal(label) == 0) {
4153 return false;
4154 }
4155 }
4156 return true;
4157 }
4158
4159 @Override
4160 public String explainFailure() {
4161 for (String label : labels) {
4162 if (labelsCache.getLabelOrdinal(label) == 0) {
4163 return label + " is not available yet";
4164 }
4165 }
4166 return "";
4167 }
4168 });
4169 }
4170
4171
4172
4173
4174
4175
4176 public static List<HColumnDescriptor> generateColumnDescriptors() {
4177 return generateColumnDescriptors("");
4178 }
4179
4180
4181
4182
4183
4184
4185
4186 public static List<HColumnDescriptor> generateColumnDescriptors(final String prefix) {
4187 List<HColumnDescriptor> htds = new ArrayList<HColumnDescriptor>();
4188 long familyId = 0;
4189 for (Compression.Algorithm compressionType: getSupportedCompressionAlgorithms()) {
4190 for (DataBlockEncoding encodingType: DataBlockEncoding.values()) {
4191 for (BloomType bloomType: BloomType.values()) {
4192 String name = String.format("%s-cf-!@#&-%d!@#", prefix, familyId);
4193 HColumnDescriptor htd = new HColumnDescriptor(name);
4194 htd.setCompressionType(compressionType);
4195 htd.setDataBlockEncoding(encodingType);
4196 htd.setBloomFilterType(bloomType);
4197 htds.add(htd);
4198 familyId++;
4199 }
4200 }
4201 }
4202 return htds;
4203 }
4204
4205
4206
4207
4208
4209 public static Compression.Algorithm[] getSupportedCompressionAlgorithms() {
4210 String[] allAlgos = HFile.getSupportedCompressionAlgorithms();
4211 List<Compression.Algorithm> supportedAlgos = new ArrayList<Compression.Algorithm>();
4212 for (String algoName : allAlgos) {
4213 try {
4214 Compression.Algorithm algo = Compression.getCompressionAlgorithmByName(algoName);
4215 algo.getCompressor();
4216 supportedAlgos.add(algo);
4217 } catch (Throwable t) {
4218
4219 }
4220 }
4221 return supportedAlgos.toArray(new Algorithm[supportedAlgos.size()]);
4222 }
4223
4224
4225
4226
4227
4228
4229
4230 public MiniKdc setupMiniKdc(File keytabFile) throws Exception {
4231 Properties conf = MiniKdc.createConf();
4232 conf.put(MiniKdc.DEBUG, true);
4233 MiniKdc kdc = null;
4234 File dir = null;
4235
4236
4237 boolean bindException;
4238 int numTries = 0;
4239 do {
4240 try {
4241 bindException = false;
4242 dir = new File(getDataTestDir("kdc").toUri().getPath());
4243 kdc = new MiniKdc(conf, dir);
4244 kdc.start();
4245 } catch (BindException e) {
4246 FileUtils.deleteDirectory(dir);
4247 numTries++;
4248 if (numTries == 3) {
4249 LOG.error("Failed setting up MiniKDC. Tried " + numTries + " times.");
4250 throw e;
4251 }
4252 LOG.error("BindException encountered when setting up MiniKdc. Trying again.");
4253 bindException = true;
4254 }
4255 } while (bindException);
4256 HBaseKerberosUtils.setKeytabFileForTesting(keytabFile.getAbsolutePath());
4257 return kdc;
4258 }
4259 }