1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22 import static org.junit.Assert.fail;
23
24 import java.io.File;
25 import java.io.IOException;
26 import java.io.OutputStream;
27 import java.lang.reflect.Field;
28 import java.lang.reflect.Modifier;
29 import java.net.BindException;
30 import java.net.DatagramSocket;
31 import java.net.InetAddress;
32 import java.net.ServerSocket;
33 import java.net.Socket;
34 import java.net.UnknownHostException;
35 import java.security.MessageDigest;
36 import java.util.ArrayList;
37 import java.util.Arrays;
38 import java.util.Collection;
39 import java.util.Collections;
40 import java.util.HashSet;
41 import java.util.List;
42 import java.util.Map;
43 import java.util.NavigableSet;
44 import java.util.Properties;
45 import java.util.Random;
46 import java.util.Set;
47 import java.util.TreeSet;
48 import java.util.UUID;
49 import java.util.concurrent.TimeUnit;
50
51 import org.apache.commons.io.FileUtils;
52 import org.apache.commons.lang.RandomStringUtils;
53 import org.apache.commons.logging.Log;
54 import org.apache.commons.logging.LogFactory;
55 import org.apache.commons.logging.impl.Jdk14Logger;
56 import org.apache.commons.logging.impl.Log4JLogger;
57 import org.apache.hadoop.conf.Configuration;
58 import org.apache.hadoop.fs.FileSystem;
59 import org.apache.hadoop.fs.Path;
60 import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
61 import org.apache.hadoop.hbase.Waiter.Predicate;
62 import org.apache.hadoop.hbase.classification.InterfaceAudience;
63 import org.apache.hadoop.hbase.classification.InterfaceStability;
64 import org.apache.hadoop.hbase.client.Admin;
65 import org.apache.hadoop.hbase.client.Connection;
66 import org.apache.hadoop.hbase.client.ConnectionFactory;
67 import org.apache.hadoop.hbase.client.Consistency;
68 import org.apache.hadoop.hbase.client.Delete;
69 import org.apache.hadoop.hbase.client.Durability;
70 import org.apache.hadoop.hbase.client.Get;
71 import org.apache.hadoop.hbase.client.HBaseAdmin;
72 import org.apache.hadoop.hbase.client.HConnection;
73 import org.apache.hadoop.hbase.client.HTable;
74 import org.apache.hadoop.hbase.client.Put;
75 import org.apache.hadoop.hbase.client.RegionLocator;
76 import org.apache.hadoop.hbase.client.Result;
77 import org.apache.hadoop.hbase.client.ResultScanner;
78 import org.apache.hadoop.hbase.client.Scan;
79 import org.apache.hadoop.hbase.client.Table;
80 import org.apache.hadoop.hbase.fs.HFileSystem;
81 import org.apache.hadoop.hbase.io.compress.Compression;
82 import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
83 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
84 import org.apache.hadoop.hbase.io.hfile.ChecksumUtil;
85 import org.apache.hadoop.hbase.io.hfile.HFile;
86 import org.apache.hadoop.hbase.ipc.RpcServerInterface;
87 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
88 import org.apache.hadoop.hbase.mapreduce.MapreduceTestingShim;
89 import org.apache.hadoop.hbase.master.AssignmentManager;
90 import org.apache.hadoop.hbase.master.HMaster;
91 import org.apache.hadoop.hbase.master.RegionStates;
92 import org.apache.hadoop.hbase.master.ServerManager;
93 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
94 import org.apache.hadoop.hbase.regionserver.BloomType;
95 import org.apache.hadoop.hbase.regionserver.HRegion;
96 import org.apache.hadoop.hbase.regionserver.HRegionServer;
97 import org.apache.hadoop.hbase.regionserver.HStore;
98 import org.apache.hadoop.hbase.regionserver.InternalScanner;
99 import org.apache.hadoop.hbase.regionserver.Region;
100 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
101 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
102 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
103 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
104 import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
105 import org.apache.hadoop.hbase.security.User;
106 import org.apache.hadoop.hbase.security.visibility.VisibilityLabelsCache;
107 import org.apache.hadoop.hbase.util.Bytes;
108 import org.apache.hadoop.hbase.util.FSTableDescriptors;
109 import org.apache.hadoop.hbase.util.FSUtils;
110 import org.apache.hadoop.hbase.util.JVMClusterUtil;
111 import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
112 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
113 import org.apache.hadoop.hbase.util.Pair;
114 import org.apache.hadoop.hbase.util.RegionSplitter;
115 import org.apache.hadoop.hbase.util.RetryCounter;
116 import org.apache.hadoop.hbase.util.Threads;
117 import org.apache.hadoop.hbase.wal.WAL;
118 import org.apache.hadoop.hbase.wal.WALFactory;
119 import org.apache.hadoop.hbase.zookeeper.EmptyWatcher;
120 import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
121 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
122 import org.apache.hadoop.hbase.zookeeper.ZKConfig;
123 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
124 import org.apache.hadoop.hdfs.DFSClient;
125 import org.apache.hadoop.hdfs.DistributedFileSystem;
126 import org.apache.hadoop.hdfs.MiniDFSCluster;
127 import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
128 import org.apache.hadoop.mapred.JobConf;
129 import org.apache.hadoop.mapred.MiniMRCluster;
130 import org.apache.hadoop.mapred.TaskLog;
131 import org.apache.hadoop.minikdc.MiniKdc;
132 import org.apache.zookeeper.KeeperException;
133 import org.apache.zookeeper.KeeperException.NodeExistsException;
134 import org.apache.zookeeper.WatchedEvent;
135 import org.apache.zookeeper.ZooKeeper;
136 import org.apache.zookeeper.ZooKeeper.States;
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152 @InterfaceAudience.Public
153 @InterfaceStability.Evolving
154 @SuppressWarnings("deprecation")
155 public class HBaseTestingUtility extends HBaseCommonTestingUtility {
156 private MiniZooKeeperCluster zkCluster = null;
157
158 public static final String REGIONS_PER_SERVER_KEY = "hbase.test.regions-per-server";
159
160
161
162
163 public static final int DEFAULT_REGIONS_PER_SERVER = 3;
164
165
166 public static final String PRESPLIT_TEST_TABLE_KEY = "hbase.test.pre-split-table";
167 public static final boolean PRESPLIT_TEST_TABLE = true;
168
169 public static final String USE_LOCAL_FILESYSTEM = "hbase.test.local.fileSystem";
170
171
172
173
174 private boolean passedZkCluster = false;
175 private MiniDFSCluster dfsCluster = null;
176
177 private volatile HBaseCluster hbaseCluster = null;
178 private MiniMRCluster mrCluster = null;
179
180
181 private volatile boolean miniClusterRunning;
182
183 private String hadoopLogDir;
184
185
186 private File clusterTestDir = null;
187
188
189
190 private Path dataTestDirOnTestFS = null;
191
192
193
194
195 private volatile Connection connection;
196
197
198
199
200
201
202
203
204 @Deprecated
205 private static final String TEST_DIRECTORY_KEY = "test.build.data";
206
207
208 private static String FS_URI;
209
210
211 private static final Set<Integer> takenRandomPorts = new HashSet<Integer>();
212
213
214 public static final List<Object[]> COMPRESSION_ALGORITHMS_PARAMETERIZED =
215 Arrays.asList(new Object[][] {
216 { Compression.Algorithm.NONE },
217 { Compression.Algorithm.GZ }
218 });
219
220
221 public static final List<Object[]> BOOLEAN_PARAMETERIZED =
222 Arrays.asList(new Object[][] {
223 { new Boolean(false) },
224 { new Boolean(true) }
225 });
226
227
228 public static final List<Object[]> MEMSTORETS_TAGS_PARAMETRIZED = memStoreTSAndTagsCombination() ;
229
230 public static final Compression.Algorithm[] COMPRESSION_ALGORITHMS ={
231 Compression.Algorithm.NONE, Compression.Algorithm.GZ
232 };
233
234
235
236
237
238
239
240 public static boolean available(int port) {
241 ServerSocket ss = null;
242 DatagramSocket ds = null;
243 try {
244 ss = new ServerSocket(port);
245 ss.setReuseAddress(true);
246 ds = new DatagramSocket(port);
247 ds.setReuseAddress(true);
248 return true;
249 } catch (IOException e) {
250
251 } finally {
252 if (ds != null) {
253 ds.close();
254 }
255
256 if (ss != null) {
257 try {
258 ss.close();
259 } catch (IOException e) {
260
261 }
262 }
263 }
264
265 return false;
266 }
267
268
269
270
271
272 private static List<Object[]> bloomAndCompressionCombinations() {
273 List<Object[]> configurations = new ArrayList<Object[]>();
274 for (Compression.Algorithm comprAlgo :
275 HBaseTestingUtility.COMPRESSION_ALGORITHMS) {
276 for (BloomType bloomType : BloomType.values()) {
277 configurations.add(new Object[] { comprAlgo, bloomType });
278 }
279 }
280 return Collections.unmodifiableList(configurations);
281 }
282
283
284
285
286 private static List<Object[]> memStoreTSAndTagsCombination() {
287 List<Object[]> configurations = new ArrayList<Object[]>();
288 configurations.add(new Object[] { false, false });
289 configurations.add(new Object[] { false, true });
290 configurations.add(new Object[] { true, false });
291 configurations.add(new Object[] { true, true });
292 return Collections.unmodifiableList(configurations);
293 }
294
295 public static final Collection<Object[]> BLOOM_AND_COMPRESSION_COMBINATIONS =
296 bloomAndCompressionCombinations();
297
298 public HBaseTestingUtility() {
299 this(HBaseConfiguration.create());
300 }
301
302 public HBaseTestingUtility(Configuration conf) {
303 super(conf);
304
305
306 ChecksumUtil.generateExceptionForChecksumFailureForTest(true);
307 }
308
309
310
311
312
313
314
315 public static HBaseTestingUtility createLocalHTU() {
316 Configuration c = HBaseConfiguration.create();
317 return createLocalHTU(c);
318 }
319
320
321
322
323
324
325
326
327 public static HBaseTestingUtility createLocalHTU(Configuration c) {
328 HBaseTestingUtility htu = new HBaseTestingUtility(c);
329 String dataTestDir = htu.getDataTestDir().toString();
330 htu.getConfiguration().set(HConstants.HBASE_DIR, dataTestDir);
331 LOG.debug("Setting " + HConstants.HBASE_DIR + " to " + dataTestDir);
332 return htu;
333 }
334
335
336
337
338 public static void closeRegion(final Region r) throws IOException {
339 if (r != null) {
340 ((HRegion)r).close();
341 }
342 }
343
344
345
346
347
348
349
350
351
352
353
354
355 @Override
356 public Configuration getConfiguration() {
357 return super.getConfiguration();
358 }
359
360 public void setHBaseCluster(HBaseCluster hbaseCluster) {
361 this.hbaseCluster = hbaseCluster;
362 }
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380 @Override
381 protected Path setupDataTestDir() {
382 Path testPath = super.setupDataTestDir();
383 if (null == testPath) {
384 return null;
385 }
386
387 createSubDirAndSystemProperty(
388 "hadoop.log.dir",
389 testPath, "hadoop-log-dir");
390
391
392
393 createSubDirAndSystemProperty(
394 "hadoop.tmp.dir",
395 testPath, "hadoop-tmp-dir");
396
397
398 createSubDir(
399 "mapreduce.cluster.local.dir",
400 testPath, "mapred-local-dir");
401
402 return testPath;
403 }
404
405 public void setJobWithoutMRCluster() throws IOException {
406 conf.set("hbase.fs.tmp.dir", getDataTestDirOnTestFS("hbase-staging").toString());
407 conf.setBoolean(HBaseTestingUtility.USE_LOCAL_FILESYSTEM, true);
408 }
409
410 private void createSubDirAndSystemProperty(
411 String propertyName, Path parent, String subDirName){
412
413 String sysValue = System.getProperty(propertyName);
414
415 if (sysValue != null) {
416
417
418 LOG.info("System.getProperty(\""+propertyName+"\") already set to: "+
419 sysValue + " so I do NOT create it in " + parent);
420 String confValue = conf.get(propertyName);
421 if (confValue != null && !confValue.endsWith(sysValue)){
422 LOG.warn(
423 propertyName + " property value differs in configuration and system: "+
424 "Configuration="+confValue+" while System="+sysValue+
425 " Erasing configuration value by system value."
426 );
427 }
428 conf.set(propertyName, sysValue);
429 } else {
430
431 createSubDir(propertyName, parent, subDirName);
432 System.setProperty(propertyName, conf.get(propertyName));
433 }
434 }
435
436
437
438
439
440
441
442 private Path getBaseTestDirOnTestFS() throws IOException {
443 FileSystem fs = getTestFileSystem();
444 return new Path(fs.getWorkingDirectory(), "test-data");
445 }
446
447
448
449
450 public HTableDescriptor getMetaTableDescriptor() {
451 try {
452 return new FSTableDescriptors(conf).get(TableName.META_TABLE_NAME);
453 } catch (IOException e) {
454 throw new RuntimeException("Unable to create META table descriptor", e);
455 }
456 }
457
458
459
460
461
462
463 Path getClusterTestDir() {
464 if (clusterTestDir == null){
465 setupClusterTestDir();
466 }
467 return new Path(clusterTestDir.getAbsolutePath());
468 }
469
470
471
472
473 private void setupClusterTestDir() {
474 if (clusterTestDir != null) {
475 return;
476 }
477
478
479
480 Path testDir = getDataTestDir("dfscluster_" + UUID.randomUUID().toString());
481 clusterTestDir = new File(testDir.toString()).getAbsoluteFile();
482
483 boolean b = deleteOnExit();
484 if (b) clusterTestDir.deleteOnExit();
485 conf.set(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
486 LOG.info("Created new mini-cluster data directory: " + clusterTestDir + ", deleteOnExit=" + b);
487 }
488
489
490
491
492
493
494
495 public Path getDataTestDirOnTestFS() throws IOException {
496 if (dataTestDirOnTestFS == null) {
497 setupDataTestDirOnTestFS();
498 }
499
500 return dataTestDirOnTestFS;
501 }
502
503
504
505
506
507
508
509
510 public Path getDataTestDirOnTestFS(final String subdirName) throws IOException {
511 return new Path(getDataTestDirOnTestFS(), subdirName);
512 }
513
514
515
516
517
518 private void setupDataTestDirOnTestFS() throws IOException {
519 if (dataTestDirOnTestFS != null) {
520 LOG.warn("Data test on test fs dir already setup in "
521 + dataTestDirOnTestFS.toString());
522 return;
523 }
524 dataTestDirOnTestFS = getNewDataTestDirOnTestFS();
525 }
526
527
528
529
530 private Path getNewDataTestDirOnTestFS() throws IOException {
531
532
533
534
535 FileSystem fs = getTestFileSystem();
536 Path newDataTestDir = null;
537 if (fs.getUri().getScheme().equals(FileSystem.getLocal(conf).getUri().getScheme())) {
538 File dataTestDir = new File(getDataTestDir().toString());
539 if (deleteOnExit()) dataTestDir.deleteOnExit();
540 newDataTestDir = new Path(dataTestDir.getAbsolutePath());
541 } else {
542 Path base = getBaseTestDirOnTestFS();
543 String randomStr = UUID.randomUUID().toString();
544 newDataTestDir = new Path(base, randomStr);
545 if (deleteOnExit()) fs.deleteOnExit(newDataTestDir);
546 }
547 return newDataTestDir;
548 }
549
550
551
552
553
554
555 public boolean cleanupDataTestDirOnTestFS() throws IOException {
556 boolean ret = getTestFileSystem().delete(dataTestDirOnTestFS, true);
557 if (ret)
558 dataTestDirOnTestFS = null;
559 return ret;
560 }
561
562
563
564
565
566
567 public boolean cleanupDataTestDirOnTestFS(String subdirName) throws IOException {
568 Path cpath = getDataTestDirOnTestFS(subdirName);
569 return getTestFileSystem().delete(cpath, true);
570 }
571
572
573
574
575
576
577
578
579 public MiniDFSCluster startMiniDFSCluster(int servers) throws Exception {
580 return startMiniDFSCluster(servers, null);
581 }
582
583
584
585
586
587
588
589
590
591
592
593
594 public MiniDFSCluster startMiniDFSCluster(final String hosts[])
595 throws Exception {
596 if ( hosts != null && hosts.length != 0) {
597 return startMiniDFSCluster(hosts.length, hosts);
598 } else {
599 return startMiniDFSCluster(1, null);
600 }
601 }
602
603
604
605
606
607
608
609
610
611
612 public MiniDFSCluster startMiniDFSCluster(int servers, final String hosts[])
613 throws Exception {
614 createDirsAndSetProperties();
615 EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
616
617
618 org.apache.log4j.Logger.getLogger(org.apache.hadoop.metrics2.util.MBeans.class).
619 setLevel(org.apache.log4j.Level.ERROR);
620 org.apache.log4j.Logger.getLogger(org.apache.hadoop.metrics2.impl.MetricsSystemImpl.class).
621 setLevel(org.apache.log4j.Level.ERROR);
622
623
624 this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true,
625 true, null, null, hosts, null);
626
627
628 setFs();
629
630
631 this.dfsCluster.waitClusterUp();
632
633
634 dataTestDirOnTestFS = null;
635
636 return this.dfsCluster;
637 }
638
639 private void setFs() throws IOException {
640 if(this.dfsCluster == null){
641 LOG.info("Skipping setting fs because dfsCluster is null");
642 return;
643 }
644 FileSystem fs = this.dfsCluster.getFileSystem();
645 FSUtils.setFsDefault(this.conf, new Path(fs.getUri()));
646 if (this.conf.getBoolean(USE_LOCAL_FILESYSTEM, false)) {
647 FSUtils.setFsDefault(this.conf, new Path("file:///"));
648 }
649 }
650
651 public MiniDFSCluster startMiniDFSCluster(int servers, final String racks[], String hosts[])
652 throws Exception {
653 createDirsAndSetProperties();
654 this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true,
655 true, null, racks, hosts, null);
656
657
658 FileSystem fs = this.dfsCluster.getFileSystem();
659 FSUtils.setFsDefault(this.conf, new Path(fs.getUri()));
660
661
662 this.dfsCluster.waitClusterUp();
663
664
665 dataTestDirOnTestFS = null;
666
667 return this.dfsCluster;
668 }
669
670 public MiniDFSCluster startMiniDFSClusterForTestWAL(int namenodePort) throws IOException {
671 createDirsAndSetProperties();
672 dfsCluster = new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null,
673 null, null, null);
674 return dfsCluster;
675 }
676
677
678 private void createDirsAndSetProperties() throws IOException {
679 setupClusterTestDir();
680 System.setProperty(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
681 createDirAndSetProperty("cache_data", "test.cache.data");
682 createDirAndSetProperty("hadoop_tmp", "hadoop.tmp.dir");
683 hadoopLogDir = createDirAndSetProperty("hadoop_logs", "hadoop.log.dir");
684 createDirAndSetProperty("mapred_local", "mapreduce.cluster.local.dir");
685 createDirAndSetProperty("mapred_temp", "mapreduce.cluster.temp.dir");
686 enableShortCircuit();
687
688 Path root = getDataTestDirOnTestFS("hadoop");
689 conf.set(MapreduceTestingShim.getMROutputDirProp(),
690 new Path(root, "mapred-output-dir").toString());
691 conf.set("mapreduce.jobtracker.system.dir", new Path(root, "mapred-system-dir").toString());
692 conf.set("mapreduce.jobtracker.staging.root.dir",
693 new Path(root, "mapreduce-jobtracker-staging-root-dir").toString());
694 conf.set("mapreduce.job.working.dir", new Path(root, "mapred-working-dir").toString());
695 }
696
697
698
699
700
701
702
703 public boolean isReadShortCircuitOn(){
704 final String propName = "hbase.tests.use.shortcircuit.reads";
705 String readOnProp = System.getProperty(propName);
706 if (readOnProp != null){
707 return Boolean.parseBoolean(readOnProp);
708 } else {
709 return conf.getBoolean(propName, false);
710 }
711 }
712
713
714
715
716 private void enableShortCircuit() {
717 if (isReadShortCircuitOn()) {
718 String curUser = System.getProperty("user.name");
719 LOG.info("read short circuit is ON for user " + curUser);
720
721 conf.set("dfs.block.local-path-access.user", curUser);
722
723 conf.setBoolean("dfs.client.read.shortcircuit", true);
724
725 conf.setBoolean("dfs.client.read.shortcircuit.skip.checksum", true);
726 } else {
727 LOG.info("read short circuit is OFF");
728 }
729 }
730
731 private String createDirAndSetProperty(final String relPath, String property) {
732 String path = getDataTestDir(relPath).toString();
733 System.setProperty(property, path);
734 conf.set(property, path);
735 new File(path).mkdirs();
736 LOG.info("Setting " + property + " to " + path + " in system properties and HBase conf");
737 return path;
738 }
739
740
741
742
743
744
745 public void shutdownMiniDFSCluster() throws IOException {
746 if (this.dfsCluster != null) {
747
748 this.dfsCluster.shutdown();
749 dfsCluster = null;
750 dataTestDirOnTestFS = null;
751 FSUtils.setFsDefault(this.conf, new Path("file:///"));
752 }
753 }
754
755
756
757
758
759
760
761
762 public MiniZooKeeperCluster startMiniZKCluster() throws Exception {
763 return startMiniZKCluster(1);
764 }
765
766
767
768
769
770
771
772
773
774 public MiniZooKeeperCluster startMiniZKCluster(
775 final int zooKeeperServerNum,
776 final int ... clientPortList)
777 throws Exception {
778 setupClusterTestDir();
779 return startMiniZKCluster(clusterTestDir, zooKeeperServerNum, clientPortList);
780 }
781
782 private MiniZooKeeperCluster startMiniZKCluster(final File dir)
783 throws Exception {
784 return startMiniZKCluster(dir, 1, null);
785 }
786
787
788
789
790
791 private MiniZooKeeperCluster startMiniZKCluster(final File dir,
792 final int zooKeeperServerNum,
793 final int [] clientPortList)
794 throws Exception {
795 if (this.zkCluster != null) {
796 throw new IOException("Cluster already running at " + dir);
797 }
798 this.passedZkCluster = false;
799 this.zkCluster = new MiniZooKeeperCluster(this.getConfiguration());
800 final int defPort = this.conf.getInt("test.hbase.zookeeper.property.clientPort", 0);
801 if (defPort > 0){
802
803 this.zkCluster.setDefaultClientPort(defPort);
804 }
805
806 if (clientPortList != null) {
807
808 int clientPortListSize = (clientPortList.length <= zooKeeperServerNum) ?
809 clientPortList.length : zooKeeperServerNum;
810 for (int i=0; i < clientPortListSize; i++) {
811 this.zkCluster.addClientPort(clientPortList[i]);
812 }
813 }
814 int clientPort = this.zkCluster.startup(dir,zooKeeperServerNum);
815 this.conf.set(HConstants.ZOOKEEPER_CLIENT_PORT,
816 Integer.toString(clientPort));
817 return this.zkCluster;
818 }
819
820
821
822
823
824
825
826 public void shutdownMiniZKCluster() throws IOException {
827 if (this.zkCluster != null) {
828 this.zkCluster.shutdown();
829 this.zkCluster = null;
830 }
831 }
832
833
834
835
836
837
838
839 public MiniHBaseCluster startMiniCluster() throws Exception {
840 return startMiniCluster(1, 1);
841 }
842
843
844
845
846
847
848
849 public MiniHBaseCluster startMiniCluster(boolean withWALDir) throws Exception {
850 return startMiniCluster(1, 1, 1, null, null, null, false, withWALDir);
851 }
852
853
854
855
856
857
858
859
860
861 public MiniHBaseCluster startMiniCluster(final int numSlaves, boolean create)
862 throws Exception {
863 return startMiniCluster(1, numSlaves, create);
864 }
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879 public MiniHBaseCluster startMiniCluster(final int numSlaves)
880 throws Exception {
881 return startMiniCluster(1, numSlaves, false);
882 }
883
884 public MiniHBaseCluster startMiniCluster(final int numSlaves, boolean create, boolean withWALDir)
885 throws Exception {
886 return startMiniCluster(1, numSlaves, numSlaves, null, null, null, create, withWALDir);
887 }
888
889
890
891
892
893
894
895
896 public MiniHBaseCluster startMiniCluster(final int numMasters,
897 final int numSlaves, boolean create)
898 throws Exception {
899 return startMiniCluster(numMasters, numSlaves, null, create);
900 }
901
902
903
904
905
906
907
908 public MiniHBaseCluster startMiniCluster(final int numMasters,
909 final int numSlaves)
910 throws Exception {
911 return startMiniCluster(numMasters, numSlaves, null, false);
912 }
913
914 public MiniHBaseCluster startMiniCluster(final int numMasters,
915 final int numSlaves, final String[] dataNodeHosts, boolean create)
916 throws Exception {
917 return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts,
918 null, null, create, false);
919 }
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945 public MiniHBaseCluster startMiniCluster(final int numMasters,
946 final int numSlaves, final String[] dataNodeHosts) throws Exception {
947 return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts,
948 null, null);
949 }
950
951
952
953
954
955 public MiniHBaseCluster startMiniCluster(final int numMasters,
956 final int numSlaves, final int numDataNodes) throws Exception {
957 return startMiniCluster(numMasters, numSlaves, numDataNodes, null, null, null);
958 }
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987 public MiniHBaseCluster startMiniCluster(final int numMasters,
988 final int numSlaves, final String[] dataNodeHosts, Class<? extends HMaster> masterClass,
989 Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
990 throws Exception {
991 return startMiniCluster(
992 numMasters, numSlaves, numSlaves, dataNodeHosts, masterClass, regionserverClass);
993 }
994
995 public MiniHBaseCluster startMiniCluster(final int numMasters,
996 final int numSlaves, int numDataNodes, final String[] dataNodeHosts,
997 Class<? extends HMaster> masterClass,
998 Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
999 throws Exception {
1000 return startMiniCluster(numMasters, numSlaves, numDataNodes, dataNodeHosts,
1001 masterClass, regionserverClass, false, false);
1002 }
1003
1004
1005
1006
1007
1008
1009
1010
1011 public MiniHBaseCluster startMiniCluster(final int numMasters,
1012 final int numSlaves, int numDataNodes, final String[] dataNodeHosts,
1013 Class<? extends HMaster> masterClass,
1014 Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass,
1015 boolean create, boolean withWALDir)
1016 throws Exception {
1017 if (dataNodeHosts != null && dataNodeHosts.length != 0) {
1018 numDataNodes = dataNodeHosts.length;
1019 }
1020
1021 LOG.info("Starting up minicluster with " + numMasters + " master(s) and " +
1022 numSlaves + " regionserver(s) and " + numDataNodes + " datanode(s)");
1023
1024
1025 if (miniClusterRunning) {
1026 throw new IllegalStateException("A mini-cluster is already running");
1027 }
1028 miniClusterRunning = true;
1029
1030 setupClusterTestDir();
1031 System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestDir.getPath());
1032
1033
1034
1035 if(this.dfsCluster == null) {
1036 dfsCluster = startMiniDFSCluster(numDataNodes, dataNodeHosts);
1037 }
1038
1039
1040 if (this.zkCluster == null) {
1041 startMiniZKCluster(clusterTestDir);
1042 }
1043
1044
1045 return startMiniHBaseCluster(numMasters, numSlaves, masterClass,
1046 regionserverClass, create, withWALDir);
1047 }
1048
1049 public MiniHBaseCluster startMiniHBaseCluster(final int numMasters, final int numSlaves)
1050 throws IOException, InterruptedException{
1051 return startMiniHBaseCluster(numMasters, numSlaves, null, null, false, false);
1052 }
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067 public MiniHBaseCluster startMiniHBaseCluster(final int numMasters,
1068 final int numSlaves, Class<? extends HMaster> masterClass,
1069 Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass,
1070 boolean create, boolean withWALDir)
1071 throws IOException, InterruptedException {
1072
1073 createRootDir(create);
1074
1075 if (withWALDir) {
1076 createWALRootDir();
1077 }
1078
1079
1080
1081 if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1) == -1) {
1082 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, numSlaves);
1083 }
1084 if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1) == -1) {
1085 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, numSlaves);
1086 }
1087
1088 Configuration c = new Configuration(this.conf);
1089 this.hbaseCluster =
1090 new MiniHBaseCluster(c, numMasters, numSlaves, masterClass, regionserverClass);
1091
1092 Table t = new HTable(c, TableName.META_TABLE_NAME);
1093 ResultScanner s = t.getScanner(new Scan());
1094 while (s.next() != null) {
1095 continue;
1096 }
1097 s.close();
1098 t.close();
1099
1100 getHBaseAdmin();
1101 LOG.info("Minicluster is up");
1102
1103
1104
1105 setHBaseFsTmpDir();
1106
1107 return (MiniHBaseCluster)this.hbaseCluster;
1108 }
1109
1110
1111
1112
1113
1114
1115
1116 public void restartHBaseCluster(int servers) throws IOException, InterruptedException {
1117 this.hbaseCluster = new MiniHBaseCluster(this.conf, servers);
1118
1119 Table t = new HTable(new Configuration(this.conf), TableName.META_TABLE_NAME);
1120 ResultScanner s = t.getScanner(new Scan());
1121 while (s.next() != null) {
1122
1123 }
1124 LOG.info("HBase has been restarted");
1125 s.close();
1126 t.close();
1127 }
1128
1129
1130
1131
1132
1133
1134 public MiniHBaseCluster getMiniHBaseCluster() {
1135 if (this.hbaseCluster == null || this.hbaseCluster instanceof MiniHBaseCluster) {
1136 return (MiniHBaseCluster)this.hbaseCluster;
1137 }
1138 throw new RuntimeException(hbaseCluster + " not an instance of " +
1139 MiniHBaseCluster.class.getName());
1140 }
1141
1142
1143
1144
1145
1146
1147 public void shutdownMiniCluster() throws Exception {
1148 LOG.info("Shutting down minicluster");
1149 if (this.connection != null && !this.connection.isClosed()) {
1150 this.connection.close();
1151 this.connection = null;
1152 }
1153 shutdownMiniHBaseCluster();
1154 if (!this.passedZkCluster){
1155 shutdownMiniZKCluster();
1156 }
1157 shutdownMiniDFSCluster();
1158
1159 cleanupTestDir();
1160 miniClusterRunning = false;
1161 LOG.info("Minicluster is down");
1162 }
1163
1164
1165
1166
1167
1168 @Override
1169 public boolean cleanupTestDir() throws IOException {
1170 boolean ret = super.cleanupTestDir();
1171 if (deleteDir(this.clusterTestDir)) {
1172 this.clusterTestDir = null;
1173 return ret & true;
1174 }
1175 return false;
1176 }
1177
1178
1179
1180
1181
1182 public void shutdownMiniHBaseCluster() throws IOException {
1183 if (hbaseAdmin != null) {
1184 hbaseAdmin.close0();
1185 hbaseAdmin = null;
1186 }
1187
1188
1189 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1);
1190 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1);
1191 if (this.hbaseCluster != null) {
1192 this.hbaseCluster.shutdown();
1193
1194 this.hbaseCluster.waitUntilShutDown();
1195 this.hbaseCluster = null;
1196 }
1197
1198 if (zooKeeperWatcher != null) {
1199 zooKeeperWatcher.close();
1200 zooKeeperWatcher = null;
1201 }
1202 }
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212 public Path getDefaultRootDirPath(boolean create) throws IOException {
1213 if (!create) {
1214 return getDataTestDirOnTestFS();
1215 } else {
1216 return getNewDataTestDirOnTestFS();
1217 }
1218 }
1219
1220
1221
1222
1223
1224
1225
1226
1227 public Path getDefaultRootDirPath() throws IOException {
1228 return getDefaultRootDirPath(false);
1229 }
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243 public Path createRootDir(boolean create) throws IOException {
1244 FileSystem fs = FileSystem.get(this.conf);
1245 Path hbaseRootdir = getDefaultRootDirPath(create);
1246 FSUtils.setRootDir(this.conf, hbaseRootdir);
1247 fs.mkdirs(hbaseRootdir);
1248 FSUtils.setVersion(fs, hbaseRootdir);
1249 return hbaseRootdir;
1250 }
1251
1252
1253
1254
1255
1256
1257
1258 public Path createRootDir() throws IOException {
1259 return createRootDir(false);
1260 }
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271 public Path createWALRootDir() throws IOException {
1272 FileSystem fs = FileSystem.get(this.conf);
1273 Path walDir = getNewDataTestDirOnTestFS();
1274 FSUtils.setWALRootDir(this.conf, walDir);
1275 fs.mkdirs(walDir);
1276 return walDir;
1277 }
1278
1279 private void setHBaseFsTmpDir() throws IOException {
1280 String hbaseFsTmpDirInString = this.conf.get("hbase.fs.tmp.dir");
1281 if (hbaseFsTmpDirInString == null) {
1282 this.conf.set("hbase.fs.tmp.dir", getDataTestDirOnTestFS("hbase-staging").toString());
1283 LOG.info("Setting hbase.fs.tmp.dir to " + this.conf.get("hbase.fs.tmp.dir"));
1284 } else {
1285 LOG.info("The hbase.fs.tmp.dir is set to " + hbaseFsTmpDirInString);
1286 }
1287 }
1288
1289
1290
1291
1292
1293 public void flush() throws IOException {
1294 getMiniHBaseCluster().flushcache();
1295 }
1296
1297
1298
1299
1300
1301 public void flush(TableName tableName) throws IOException {
1302 getMiniHBaseCluster().flushcache(tableName);
1303 }
1304
1305
1306
1307
1308
1309 public void compact(boolean major) throws IOException {
1310 getMiniHBaseCluster().compact(major);
1311 }
1312
1313
1314
1315
1316
1317 public void compact(TableName tableName, boolean major) throws IOException {
1318 getMiniHBaseCluster().compact(tableName, major);
1319 }
1320
1321
1322
1323
1324
1325
1326
1327
1328 public Table createTable(TableName tableName, String family)
1329 throws IOException{
1330 return createTable(tableName, new String[]{family});
1331 }
1332
1333
1334
1335
1336
1337
1338
1339
1340 public HTable createTable(byte[] tableName, byte[] family)
1341 throws IOException{
1342 return createTable(TableName.valueOf(tableName), new byte[][]{family});
1343 }
1344
1345
1346
1347
1348
1349
1350
1351
1352 public Table createTable(TableName tableName, String[] families)
1353 throws IOException {
1354 List<byte[]> fams = new ArrayList<byte[]>(families.length);
1355 for (String family : families) {
1356 fams.add(Bytes.toBytes(family));
1357 }
1358 return createTable(tableName, fams.toArray(new byte[0][]));
1359 }
1360
1361
1362
1363
1364
1365
1366
1367
1368 public HTable createTable(TableName tableName, byte[] family)
1369 throws IOException{
1370 return createTable(tableName, new byte[][]{family});
1371 }
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381 public HTable createMultiRegionTable(TableName tableName, byte[] family, int numRegions)
1382 throws IOException {
1383 if (numRegions < 3) throw new IOException("Must create at least 3 regions");
1384 byte[] startKey = Bytes.toBytes("aaaaa");
1385 byte[] endKey = Bytes.toBytes("zzzzz");
1386 byte[][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
1387
1388 return createTable(tableName, new byte[][] { family }, splitKeys);
1389 }
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399 public HTable createTable(byte[] tableName, byte[][] families)
1400 throws IOException {
1401 return createTable(tableName, families,
1402 new Configuration(getConfiguration()));
1403 }
1404
1405
1406
1407
1408
1409
1410
1411
1412 public HTable createTable(TableName tableName, byte[][] families)
1413 throws IOException {
1414 return createTable(tableName, families, (byte[][]) null);
1415 }
1416
1417
1418
1419
1420
1421
1422
1423
1424 public HTable createMultiRegionTable(TableName tableName, byte[][] families) throws IOException {
1425 return createTable(tableName, families, KEYS_FOR_HBA_CREATE_TABLE);
1426 }
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436 public HTable createTable(TableName tableName, byte[][] families, byte[][] splitKeys)
1437 throws IOException {
1438 return createTable(tableName, families, splitKeys, new Configuration(getConfiguration()));
1439 }
1440
1441 public HTable createTable(byte[] tableName, byte[][] families,
1442 int numVersions, byte[] startKey, byte[] endKey, int numRegions) throws IOException {
1443 return createTable(TableName.valueOf(tableName), families, numVersions,
1444 startKey, endKey, numRegions);
1445 }
1446
1447 public HTable createTable(String tableName, byte[][] families,
1448 int numVersions, byte[] startKey, byte[] endKey, int numRegions) throws IOException {
1449 return createTable(TableName.valueOf(tableName), families, numVersions,
1450 startKey, endKey, numRegions);
1451 }
1452
1453 public HTable createTable(TableName tableName, byte[][] families,
1454 int numVersions, byte[] startKey, byte[] endKey, int numRegions)
1455 throws IOException{
1456 HTableDescriptor desc = new HTableDescriptor(tableName);
1457 for (byte[] family : families) {
1458 HColumnDescriptor hcd = new HColumnDescriptor(family)
1459 .setMaxVersions(numVersions);
1460 desc.addFamily(hcd);
1461 }
1462 getHBaseAdmin().createTable(desc, startKey, endKey, numRegions);
1463
1464 waitUntilAllRegionsAssigned(tableName);
1465 return new HTable(getConfiguration(), tableName);
1466 }
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476 public HTable createTable(HTableDescriptor htd, byte[][] families, Configuration c)
1477 throws IOException {
1478 return createTable(htd, families, (byte[][]) null, c);
1479 }
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490 public HTable createTable(HTableDescriptor htd, byte[][] families, byte[][] splitKeys,
1491 Configuration c) throws IOException {
1492 for (byte[] family : families) {
1493 HColumnDescriptor hcd = new HColumnDescriptor(family);
1494
1495
1496
1497 hcd.setBloomFilterType(BloomType.NONE);
1498 htd.addFamily(hcd);
1499 }
1500 getHBaseAdmin().createTable(htd, splitKeys);
1501
1502
1503 waitUntilAllRegionsAssigned(htd.getTableName());
1504 return (HTable) getConnection().getTable(htd.getTableName());
1505 }
1506
1507
1508
1509
1510
1511
1512
1513
1514 public HTable createTable(HTableDescriptor htd, byte[][] splitRows)
1515 throws IOException {
1516 getHBaseAdmin().createTable(htd, splitRows);
1517
1518 waitUntilAllRegionsAssigned(htd.getTableName());
1519 return new HTable(getConfiguration(), htd.getTableName());
1520 }
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530 public HTable createTable(TableName tableName, byte[][] families,
1531 final Configuration c)
1532 throws IOException {
1533 return createTable(tableName, families, (byte[][]) null, c);
1534 }
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545 public HTable createTable(TableName tableName, byte[][] families, byte[][] splitKeys,
1546 final Configuration c) throws IOException {
1547 return createTable(new HTableDescriptor(tableName), families, splitKeys, c);
1548 }
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558 public HTable createTable(byte[] tableName, byte[][] families,
1559 final Configuration c)
1560 throws IOException {
1561 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
1562 for(byte[] family : families) {
1563 HColumnDescriptor hcd = new HColumnDescriptor(family);
1564
1565
1566
1567 hcd.setBloomFilterType(BloomType.NONE);
1568 desc.addFamily(hcd);
1569 }
1570 getHBaseAdmin().createTable(desc);
1571 return new HTable(c, desc.getTableName());
1572 }
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583 public HTable createTable(TableName tableName, byte[][] families,
1584 final Configuration c, int numVersions)
1585 throws IOException {
1586 HTableDescriptor desc = new HTableDescriptor(tableName);
1587 for(byte[] family : families) {
1588 HColumnDescriptor hcd = new HColumnDescriptor(family)
1589 .setMaxVersions(numVersions);
1590 desc.addFamily(hcd);
1591 }
1592 getHBaseAdmin().createTable(desc);
1593
1594 waitUntilAllRegionsAssigned(tableName);
1595 return new HTable(c, tableName);
1596 }
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607 public HTable createTable(byte[] tableName, byte[][] families,
1608 final Configuration c, int numVersions)
1609 throws IOException {
1610 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
1611 for(byte[] family : families) {
1612 HColumnDescriptor hcd = new HColumnDescriptor(family)
1613 .setMaxVersions(numVersions);
1614 desc.addFamily(hcd);
1615 }
1616 getHBaseAdmin().createTable(desc);
1617 return new HTable(c, desc.getTableName());
1618 }
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628 public HTable createTable(byte[] tableName, byte[] family, int numVersions)
1629 throws IOException {
1630 return createTable(tableName, new byte[][]{family}, numVersions);
1631 }
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641 public HTable createTable(TableName tableName, byte[] family, int numVersions)
1642 throws IOException {
1643 return createTable(tableName, new byte[][]{family}, numVersions);
1644 }
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654 public HTable createTable(byte[] tableName, byte[][] families,
1655 int numVersions)
1656 throws IOException {
1657 return createTable(TableName.valueOf(tableName), families, numVersions);
1658 }
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668 public HTable createTable(TableName tableName, byte[][] families,
1669 int numVersions)
1670 throws IOException {
1671 return createTable(tableName, families, numVersions, (byte[][]) null);
1672 }
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683 public HTable createTable(TableName tableName, byte[][] families, int numVersions,
1684 byte[][] splitKeys) throws IOException {
1685 HTableDescriptor desc = new HTableDescriptor(tableName);
1686 for (byte[] family : families) {
1687 HColumnDescriptor hcd = new HColumnDescriptor(family).setMaxVersions(numVersions);
1688 desc.addFamily(hcd);
1689 }
1690 getHBaseAdmin().createTable(desc, splitKeys);
1691
1692 waitUntilAllRegionsAssigned(tableName);
1693 return new HTable(new Configuration(getConfiguration()), tableName);
1694 }
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704 public HTable createMultiRegionTable(TableName tableName, byte[][] families, int numVersions)
1705 throws IOException {
1706 return createTable(tableName, families, numVersions, KEYS_FOR_HBA_CREATE_TABLE);
1707 }
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718 public HTable createTable(byte[] tableName, byte[][] families,
1719 int numVersions, int blockSize) throws IOException {
1720 return createTable(TableName.valueOf(tableName),
1721 families, numVersions, blockSize);
1722 }
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733 public HTable createTable(TableName tableName, byte[][] families,
1734 int numVersions, int blockSize) throws IOException {
1735 HTableDescriptor desc = new HTableDescriptor(tableName);
1736 for (byte[] family : families) {
1737 HColumnDescriptor hcd = new HColumnDescriptor(family)
1738 .setMaxVersions(numVersions)
1739 .setBlocksize(blockSize);
1740 desc.addFamily(hcd);
1741 }
1742 getHBaseAdmin().createTable(desc);
1743
1744 waitUntilAllRegionsAssigned(tableName);
1745 return new HTable(new Configuration(getConfiguration()), tableName);
1746 }
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756 public HTable createTable(byte[] tableName, byte[][] families,
1757 int[] numVersions)
1758 throws IOException {
1759 return createTable(TableName.valueOf(tableName), families, numVersions);
1760 }
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770 public HTable createTable(TableName tableName, byte[][] families,
1771 int[] numVersions)
1772 throws IOException {
1773 HTableDescriptor desc = new HTableDescriptor(tableName);
1774 int i = 0;
1775 for (byte[] family : families) {
1776 HColumnDescriptor hcd = new HColumnDescriptor(family)
1777 .setMaxVersions(numVersions[i]);
1778 desc.addFamily(hcd);
1779 i++;
1780 }
1781 getHBaseAdmin().createTable(desc);
1782
1783 waitUntilAllRegionsAssigned(tableName);
1784 return new HTable(new Configuration(getConfiguration()), tableName);
1785 }
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795 public HTable createTable(byte[] tableName, byte[] family, byte[][] splitRows)
1796 throws IOException{
1797 return createTable(TableName.valueOf(tableName), family, splitRows);
1798 }
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808 public HTable createTable(TableName tableName, byte[] family, byte[][] splitRows)
1809 throws IOException {
1810 HTableDescriptor desc = new HTableDescriptor(tableName);
1811 HColumnDescriptor hcd = new HColumnDescriptor(family);
1812 desc.addFamily(hcd);
1813 getHBaseAdmin().createTable(desc, splitRows);
1814
1815 waitUntilAllRegionsAssigned(tableName);
1816 return new HTable(getConfiguration(), tableName);
1817 }
1818
1819
1820
1821
1822
1823
1824
1825
1826 public HTable createMultiRegionTable(TableName tableName, byte[] family) throws IOException {
1827 return createTable(tableName, family, KEYS_FOR_HBA_CREATE_TABLE);
1828 }
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838 public HTable createTable(byte[] tableName, byte[][] families, byte[][] splitRows)
1839 throws IOException {
1840 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
1841 for(byte[] family:families) {
1842 HColumnDescriptor hcd = new HColumnDescriptor(family);
1843 desc.addFamily(hcd);
1844 }
1845 getHBaseAdmin().createTable(desc, splitRows);
1846
1847 waitUntilAllRegionsAssigned(desc.getTableName());
1848 return new HTable(getConfiguration(), desc.getTableName());
1849 }
1850
1851
1852
1853
1854 public static WAL createWal(final Configuration conf, final Path rootDir, final Path walRootDir, final HRegionInfo hri)
1855 throws IOException {
1856
1857
1858 Configuration confForWAL = new Configuration(conf);
1859 confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
1860 FSUtils.setWALRootDir(confForWAL, walRootDir);
1861 return (new WALFactory(confForWAL,
1862 Collections.<WALActionsListener>singletonList(new MetricsWAL()),
1863 "hregion-" + RandomStringUtils.randomNumeric(8))).
1864 getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace());
1865 }
1866
1867
1868
1869
1870
1871 public static HRegion createRegionAndWAL(final HRegionInfo info, final Path rootDir,
1872 final Path walRootDir, final Configuration conf, final HTableDescriptor htd) throws IOException {
1873 return createRegionAndWAL(info, rootDir, walRootDir, conf, htd, true);
1874 }
1875
1876
1877
1878
1879
1880 public static HRegion createRegionAndWAL(final HRegionInfo info, final Path rootDir,
1881 final Path walRootDir, final Configuration conf, final HTableDescriptor htd, boolean initialize)
1882 throws IOException {
1883 WAL wal = createWal(conf, rootDir, walRootDir, info);
1884 return HRegion.createHRegion(info, rootDir, conf, htd, wal, initialize);
1885 }
1886
1887
1888
1889
1890 public static void closeRegionAndWAL(final Region r) throws IOException {
1891 closeRegionAndWAL((HRegion)r);
1892 }
1893
1894
1895
1896
1897 public static void closeRegionAndWAL(final HRegion r) throws IOException {
1898 if (r == null) return;
1899 r.close();
1900 if (r.getWAL() == null) return;
1901 r.getWAL().close();
1902 }
1903
1904
1905
1906
1907 @SuppressWarnings("serial")
1908 public static void modifyTableSync(Admin admin, HTableDescriptor desc)
1909 throws IOException, InterruptedException {
1910 admin.modifyTable(desc.getTableName(), desc);
1911 Pair<Integer, Integer> status = new Pair<Integer, Integer>() {{
1912 setFirst(0);
1913 setSecond(0);
1914 }};
1915 int i = 0;
1916 do {
1917 status = admin.getAlterStatus(desc.getTableName());
1918 if (status.getSecond() != 0) {
1919 LOG.debug(status.getSecond() - status.getFirst() + "/" + status.getSecond()
1920 + " regions updated.");
1921 Thread.sleep(1 * 1000l);
1922 } else {
1923 LOG.debug("All regions updated.");
1924 break;
1925 }
1926 } while (status.getFirst() != 0 && i++ < 500);
1927 if (status.getFirst() != 0) {
1928 throw new IOException("Failed to update all regions even after 500 seconds.");
1929 }
1930 }
1931
1932
1933
1934
1935 public static void setReplicas(Admin admin, TableName table, int replicaCount)
1936 throws IOException, InterruptedException {
1937 admin.disableTable(table);
1938 HTableDescriptor desc = admin.getTableDescriptor(table);
1939 desc.setRegionReplication(replicaCount);
1940 admin.modifyTable(desc.getTableName(), desc);
1941 admin.enableTable(table);
1942 }
1943
1944
1945
1946
1947
1948 public void deleteTable(String tableName) throws IOException {
1949 deleteTable(TableName.valueOf(tableName));
1950 }
1951
1952
1953
1954
1955
1956 public void deleteTable(byte[] tableName) throws IOException {
1957 deleteTable(TableName.valueOf(tableName));
1958 }
1959
1960
1961
1962
1963
1964 public void deleteTable(TableName tableName) throws IOException {
1965 try {
1966 getHBaseAdmin().disableTable(tableName);
1967 } catch (TableNotEnabledException e) {
1968 LOG.debug("Table: " + tableName + " already disabled, so just deleting it.");
1969 }
1970 getHBaseAdmin().deleteTable(tableName);
1971 }
1972
1973
1974
1975
1976
1977 public void deleteTableIfAny(TableName tableName) throws IOException {
1978 try {
1979 deleteTable(tableName);
1980 } catch (TableNotFoundException e) {
1981
1982 }
1983 }
1984
1985
1986
1987
1988
1989 public final static byte [] fam1 = Bytes.toBytes("colfamily11");
1990 public final static byte [] fam2 = Bytes.toBytes("colfamily21");
1991 public final static byte [] fam3 = Bytes.toBytes("colfamily31");
1992 public static final byte[][] COLUMNS = {fam1, fam2, fam3};
1993 private static final int MAXVERSIONS = 3;
1994
1995 public static final char FIRST_CHAR = 'a';
1996 public static final char LAST_CHAR = 'z';
1997 public static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
1998 public static final String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
1999
2000
2001
2002
2003
2004
2005
2006
2007 public HTableDescriptor createTableDescriptor(final String name,
2008 final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) {
2009 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
2010 for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
2011 htd.addFamily(new HColumnDescriptor(cfName)
2012 .setMinVersions(minVersions)
2013 .setMaxVersions(versions)
2014 .setKeepDeletedCells(keepDeleted)
2015 .setBlockCacheEnabled(false)
2016 .setTimeToLive(ttl)
2017 );
2018 }
2019 return htd;
2020 }
2021
2022
2023
2024
2025
2026
2027
2028 public HTableDescriptor createTableDescriptor(final String name) {
2029 return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
2030 MAXVERSIONS, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
2031 }
2032
2033
2034
2035
2036
2037 public HRegion createHRegion(
2038 final HRegionInfo info,
2039 final Path rootDir,
2040 final Configuration conf,
2041 final HTableDescriptor htd) throws IOException {
2042 return HRegion.createHRegion(info, rootDir, conf, htd);
2043 }
2044
2045 public HTableDescriptor createTableDescriptor(final TableName tableName,
2046 byte[] family) {
2047 return createTableDescriptor(tableName, new byte[][] {family}, 1);
2048 }
2049
2050 public HTableDescriptor createTableDescriptor(final TableName tableName,
2051 byte[][] families, int maxVersions) {
2052 HTableDescriptor desc = new HTableDescriptor(tableName);
2053 for (byte[] family : families) {
2054 HColumnDescriptor hcd = new HColumnDescriptor(family)
2055 .setMaxVersions(maxVersions);
2056 desc.addFamily(hcd);
2057 }
2058 return desc;
2059 }
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069 public HRegion createLocalHRegion(HTableDescriptor desc, byte [] startKey,
2070 byte [] endKey)
2071 throws IOException {
2072 HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
2073 return createLocalHRegion(hri, desc);
2074 }
2075
2076
2077
2078
2079
2080
2081
2082
2083 public HRegion createLocalHRegion(HRegionInfo info, HTableDescriptor desc) throws IOException {
2084 return HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), desc);
2085 }
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095 public HRegion createLocalHRegion(HRegionInfo info, HTableDescriptor desc, WAL wal)
2096 throws IOException {
2097 return HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), desc, wal);
2098 }
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112 public HRegion createLocalHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
2113 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
2114 WAL wal, byte[]... families) throws IOException {
2115 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
2116 htd.setReadOnly(isReadOnly);
2117 for (byte[] family : families) {
2118 HColumnDescriptor hcd = new HColumnDescriptor(family);
2119
2120 hcd.setMaxVersions(Integer.MAX_VALUE);
2121 htd.addFamily(hcd);
2122 }
2123 htd.setDurability(durability);
2124 HRegionInfo info = new HRegionInfo(htd.getTableName(), startKey, stopKey, false);
2125 return createLocalHRegion(info, htd, wal);
2126 }
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137 public HTable deleteTableData(byte[] tableName) throws IOException {
2138 return deleteTableData(TableName.valueOf(tableName));
2139 }
2140
2141
2142
2143
2144
2145
2146
2147
2148 public HTable deleteTableData(TableName tableName) throws IOException {
2149 HTable table = new HTable(getConfiguration(), tableName);
2150 Scan scan = new Scan();
2151 ResultScanner resScan = table.getScanner(scan);
2152 for(Result res : resScan) {
2153 Delete del = new Delete(res.getRow());
2154 table.delete(del);
2155 }
2156 resScan = table.getScanner(scan);
2157 resScan.close();
2158 return table;
2159 }
2160
2161
2162
2163
2164
2165
2166
2167
2168 public HTable truncateTable(final TableName tableName, final boolean preserveRegions)
2169 throws IOException {
2170 Admin admin = getHBaseAdmin();
2171 if (!admin.isTableDisabled(tableName)) {
2172 admin.disableTable(tableName);
2173 }
2174 admin.truncateTable(tableName, preserveRegions);
2175 return new HTable(getConfiguration(), tableName);
2176 }
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187 public HTable truncateTable(final TableName tableName) throws IOException {
2188 return truncateTable(tableName, false);
2189 }
2190
2191
2192
2193
2194
2195
2196
2197
2198 public HTable truncateTable(final byte[] tableName, final boolean preserveRegions)
2199 throws IOException {
2200 return truncateTable(TableName.valueOf(tableName), preserveRegions);
2201 }
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212 public HTable truncateTable(final byte[] tableName) throws IOException {
2213 return truncateTable(tableName, false);
2214 }
2215
2216
2217
2218
2219
2220
2221
2222
2223 public int loadTable(final Table t, final byte[] f) throws IOException {
2224 return loadTable(t, new byte[][] {f});
2225 }
2226
2227
2228
2229
2230
2231
2232
2233
2234 public int loadTable(final Table t, final byte[] f, boolean writeToWAL) throws IOException {
2235 return loadTable(t, new byte[][] {f}, null, writeToWAL);
2236 }
2237
2238
2239
2240
2241
2242
2243
2244
2245 public int loadTable(final Table t, final byte[][] f) throws IOException {
2246 return loadTable(t, f, null);
2247 }
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257 public int loadTable(final Table t, final byte[][] f, byte[] value) throws IOException {
2258 return loadTable(t, f, value, true);
2259 }
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269 public int loadTable(final Table t, final byte[][] f, byte[] value, boolean writeToWAL) throws IOException {
2270 List<Put> puts = new ArrayList<>();
2271 for (byte[] row : HBaseTestingUtility.ROWS) {
2272 Put put = new Put(row);
2273 put.setDurability(writeToWAL ? Durability.USE_DEFAULT : Durability.SKIP_WAL);
2274 for (int i = 0; i < f.length; i++) {
2275 put.add(f[i], f[i], value != null ? value : row);
2276 }
2277 puts.add(put);
2278 }
2279 t.put(puts);
2280 return puts.size();
2281 }
2282
2283
2284
2285
2286 public static class SeenRowTracker {
2287 int dim = 'z' - 'a' + 1;
2288 int[][][] seenRows = new int[dim][dim][dim];
2289 byte[] startRow;
2290 byte[] stopRow;
2291
2292 public SeenRowTracker(byte[] startRow, byte[] stopRow) {
2293 this.startRow = startRow;
2294 this.stopRow = stopRow;
2295 }
2296
2297 void reset() {
2298 for (byte[] row : ROWS) {
2299 seenRows[i(row[0])][i(row[1])][i(row[2])] = 0;
2300 }
2301 }
2302
2303 int i(byte b) {
2304 return b - 'a';
2305 }
2306
2307 public void addRow(byte[] row) {
2308 seenRows[i(row[0])][i(row[1])][i(row[2])]++;
2309 }
2310
2311
2312
2313
2314 public void validate() {
2315 for (byte b1 = 'a'; b1 <= 'z'; b1++) {
2316 for (byte b2 = 'a'; b2 <= 'z'; b2++) {
2317 for (byte b3 = 'a'; b3 <= 'z'; b3++) {
2318 int count = seenRows[i(b1)][i(b2)][i(b3)];
2319 int expectedCount = 0;
2320 if (Bytes.compareTo(new byte[] {b1,b2,b3}, startRow) >= 0
2321 && Bytes.compareTo(new byte[] {b1,b2,b3}, stopRow) < 0) {
2322 expectedCount = 1;
2323 }
2324 if (count != expectedCount) {
2325 String row = new String(new byte[] {b1,b2,b3});
2326 throw new RuntimeException("Row:" + row + " has a seen count of " + count + " instead of " + expectedCount);
2327 }
2328 }
2329 }
2330 }
2331 }
2332 }
2333
2334 public int loadRegion(final HRegion r, final byte[] f) throws IOException {
2335 return loadRegion(r, f, false);
2336 }
2337
2338 public int loadRegion(final Region r, final byte[] f) throws IOException {
2339 return loadRegion((HRegion)r, f);
2340 }
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350 public int loadRegion(final HRegion r, final byte[] f, final boolean flush)
2351 throws IOException {
2352 byte[] k = new byte[3];
2353 int rowCount = 0;
2354 for (byte b1 = 'a'; b1 <= 'z'; b1++) {
2355 for (byte b2 = 'a'; b2 <= 'z'; b2++) {
2356 for (byte b3 = 'a'; b3 <= 'z'; b3++) {
2357 k[0] = b1;
2358 k[1] = b2;
2359 k[2] = b3;
2360 Put put = new Put(k);
2361 put.setDurability(Durability.SKIP_WAL);
2362 put.add(f, null, k);
2363 if (r.getWAL() == null) {
2364 put.setDurability(Durability.SKIP_WAL);
2365 }
2366 int preRowCount = rowCount;
2367 int pause = 10;
2368 int maxPause = 1000;
2369 while (rowCount == preRowCount) {
2370 try {
2371 r.put(put);
2372 rowCount++;
2373 } catch (RegionTooBusyException e) {
2374 pause = (pause * 2 >= maxPause) ? maxPause : pause * 2;
2375 Threads.sleep(pause);
2376 }
2377 }
2378 }
2379 }
2380 if (flush) {
2381 r.flush(true);
2382 }
2383 }
2384 return rowCount;
2385 }
2386
2387 public void loadNumericRows(final Table t, final byte[] f, int startRow, int endRow)
2388 throws IOException {
2389 for (int i = startRow; i < endRow; i++) {
2390 byte[] data = Bytes.toBytes(String.valueOf(i));
2391 Put put = new Put(data);
2392 put.add(f, null, data);
2393 t.put(put);
2394 }
2395 }
2396
2397 public void verifyNumericRows(Table table, final byte[] f, int startRow, int endRow,
2398 int replicaId)
2399 throws IOException {
2400 for (int i = startRow; i < endRow; i++) {
2401 String failMsg = "Failed verification of row :" + i;
2402 byte[] data = Bytes.toBytes(String.valueOf(i));
2403 Get get = new Get(data);
2404 get.setReplicaId(replicaId);
2405 get.setConsistency(Consistency.TIMELINE);
2406 Result result = table.get(get);
2407 assertTrue(failMsg, result.containsColumn(f, null));
2408 assertEquals(failMsg, result.getColumnCells(f, null).size(), 1);
2409 Cell cell = result.getColumnLatestCell(f, null);
2410 assertTrue(failMsg,
2411 Bytes.equals(data, 0, data.length, cell.getValueArray(), cell.getValueOffset(),
2412 cell.getValueLength()));
2413 }
2414 }
2415
2416 public void verifyNumericRows(Region region, final byte[] f, int startRow, int endRow)
2417 throws IOException {
2418 verifyNumericRows((HRegion)region, f, startRow, endRow);
2419 }
2420
2421 public void verifyNumericRows(HRegion region, final byte[] f, int startRow, int endRow)
2422 throws IOException {
2423 verifyNumericRows(region, f, startRow, endRow, true);
2424 }
2425
2426 public void verifyNumericRows(Region region, final byte[] f, int startRow, int endRow,
2427 final boolean present) throws IOException {
2428 verifyNumericRows((HRegion)region, f, startRow, endRow, present);
2429 }
2430
2431 public void verifyNumericRows(HRegion region, final byte[] f, int startRow, int endRow,
2432 final boolean present) throws IOException {
2433 for (int i = startRow; i < endRow; i++) {
2434 String failMsg = "Failed verification of row :" + i;
2435 byte[] data = Bytes.toBytes(String.valueOf(i));
2436 Result result = region.get(new Get(data));
2437
2438 boolean hasResult = result != null && !result.isEmpty();
2439 assertEquals(failMsg + result, present, hasResult);
2440 if (!present) continue;
2441
2442 assertTrue(failMsg, result.containsColumn(f, null));
2443 assertEquals(failMsg, result.getColumnCells(f, null).size(), 1);
2444 Cell cell = result.getColumnLatestCell(f, null);
2445 assertTrue(failMsg,
2446 Bytes.equals(data, 0, data.length, cell.getValueArray(), cell.getValueOffset(),
2447 cell.getValueLength()));
2448 }
2449 }
2450
2451 public void deleteNumericRows(final HTable t, final byte[] f, int startRow, int endRow)
2452 throws IOException {
2453 for (int i = startRow; i < endRow; i++) {
2454 byte[] data = Bytes.toBytes(String.valueOf(i));
2455 Delete delete = new Delete(data);
2456 delete.deleteFamily(f);
2457 t.delete(delete);
2458 }
2459 }
2460
2461
2462
2463
2464 public int countRows(final Table table) throws IOException {
2465 Scan scan = new Scan();
2466 ResultScanner results = table.getScanner(scan);
2467 int count = 0;
2468 for (@SuppressWarnings("unused") Result res : results) {
2469 count++;
2470 }
2471 results.close();
2472 return count;
2473 }
2474
2475 public int countRows(final Table table, final byte[]... families) throws IOException {
2476 Scan scan = new Scan();
2477 for (byte[] family: families) {
2478 scan.addFamily(family);
2479 }
2480 ResultScanner results = table.getScanner(scan);
2481 int count = 0;
2482 for (@SuppressWarnings("unused") Result res : results) {
2483 count++;
2484 }
2485 results.close();
2486 return count;
2487 }
2488
2489
2490
2491
2492 public int countRows(final TableName tableName) throws IOException {
2493 Table table = getConnection().getTable(tableName);
2494 try {
2495 return countRows(table);
2496 } finally {
2497 table.close();
2498 }
2499 }
2500
2501
2502
2503
2504 public String checksumRows(final Table table) throws Exception {
2505 Scan scan = new Scan();
2506 ResultScanner results = table.getScanner(scan);
2507 MessageDigest digest = MessageDigest.getInstance("MD5");
2508 for (Result res : results) {
2509 digest.update(res.getRow());
2510 }
2511 results.close();
2512 return digest.toString();
2513 }
2514
2515
2516 public static final byte[][] ROWS = new byte[(int) Math.pow('z' - 'a' + 1, 3)][3];
2517 static {
2518 int i = 0;
2519 for (byte b1 = 'a'; b1 <= 'z'; b1++) {
2520 for (byte b2 = 'a'; b2 <= 'z'; b2++) {
2521 for (byte b3 = 'a'; b3 <= 'z'; b3++) {
2522 ROWS[i][0] = b1;
2523 ROWS[i][1] = b2;
2524 ROWS[i][2] = b3;
2525 i++;
2526 }
2527 }
2528 }
2529 }
2530
2531 public static final byte[][] KEYS = {
2532 HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes("bbb"),
2533 Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
2534 Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
2535 Bytes.toBytes("iii"), Bytes.toBytes("jjj"), Bytes.toBytes("kkk"),
2536 Bytes.toBytes("lll"), Bytes.toBytes("mmm"), Bytes.toBytes("nnn"),
2537 Bytes.toBytes("ooo"), Bytes.toBytes("ppp"), Bytes.toBytes("qqq"),
2538 Bytes.toBytes("rrr"), Bytes.toBytes("sss"), Bytes.toBytes("ttt"),
2539 Bytes.toBytes("uuu"), Bytes.toBytes("vvv"), Bytes.toBytes("www"),
2540 Bytes.toBytes("xxx"), Bytes.toBytes("yyy")
2541 };
2542
2543 public static final byte[][] KEYS_FOR_HBA_CREATE_TABLE = {
2544 Bytes.toBytes("bbb"),
2545 Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
2546 Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
2547 Bytes.toBytes("iii"), Bytes.toBytes("jjj"), Bytes.toBytes("kkk"),
2548 Bytes.toBytes("lll"), Bytes.toBytes("mmm"), Bytes.toBytes("nnn"),
2549 Bytes.toBytes("ooo"), Bytes.toBytes("ppp"), Bytes.toBytes("qqq"),
2550 Bytes.toBytes("rrr"), Bytes.toBytes("sss"), Bytes.toBytes("ttt"),
2551 Bytes.toBytes("uuu"), Bytes.toBytes("vvv"), Bytes.toBytes("www"),
2552 Bytes.toBytes("xxx"), Bytes.toBytes("yyy"), Bytes.toBytes("zzz")
2553 };
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565 public List<HRegionInfo> createMultiRegionsInMeta(final Configuration conf,
2566 final HTableDescriptor htd, byte [][] startKeys)
2567 throws IOException {
2568 Table meta = new HTable(conf, TableName.META_TABLE_NAME);
2569 Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR);
2570 List<HRegionInfo> newRegions = new ArrayList<HRegionInfo>(startKeys.length);
2571
2572 for (int i = 0; i < startKeys.length; i++) {
2573 int j = (i + 1) % startKeys.length;
2574 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKeys[i],
2575 startKeys[j]);
2576 MetaTableAccessor.addRegionToMeta(meta, hri);
2577 newRegions.add(hri);
2578 }
2579
2580 meta.close();
2581 return newRegions;
2582 }
2583
2584
2585
2586
2587
2588
2589 public List<byte[]> getMetaTableRows() throws IOException {
2590
2591 Table t = new HTable(new Configuration(this.conf), TableName.META_TABLE_NAME);
2592 List<byte[]> rows = new ArrayList<byte[]>();
2593 ResultScanner s = t.getScanner(new Scan());
2594 for (Result result : s) {
2595 LOG.info("getMetaTableRows: row -> " +
2596 Bytes.toStringBinary(result.getRow()));
2597 rows.add(result.getRow());
2598 }
2599 s.close();
2600 t.close();
2601 return rows;
2602 }
2603
2604
2605
2606
2607
2608
2609 public List<byte[]> getMetaTableRows(TableName tableName) throws IOException {
2610
2611 Table t = new HTable(new Configuration(this.conf), TableName.META_TABLE_NAME);
2612 List<byte[]> rows = new ArrayList<byte[]>();
2613 ResultScanner s = t.getScanner(new Scan());
2614 for (Result result : s) {
2615 HRegionInfo info = HRegionInfo.getHRegionInfo(result);
2616 if (info == null) {
2617 LOG.error("No region info for row " + Bytes.toString(result.getRow()));
2618
2619 continue;
2620 }
2621
2622 if (info.getTable().equals(tableName)) {
2623 LOG.info("getMetaTableRows: row -> " +
2624 Bytes.toStringBinary(result.getRow()) + info);
2625 rows.add(result.getRow());
2626 }
2627 }
2628 s.close();
2629 t.close();
2630 return rows;
2631 }
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644 public HRegionServer getRSForFirstRegionInTable(TableName tableName)
2645 throws IOException, InterruptedException {
2646 List<byte[]> metaRows = getMetaTableRows(tableName);
2647 if (metaRows == null || metaRows.isEmpty()) {
2648 return null;
2649 }
2650 LOG.debug("Found " + metaRows.size() + " rows for table " +
2651 tableName);
2652 byte [] firstrow = metaRows.get(0);
2653 LOG.debug("FirstRow=" + Bytes.toString(firstrow));
2654 long pause = getConfiguration().getLong(HConstants.HBASE_CLIENT_PAUSE,
2655 HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
2656 int numRetries = getConfiguration().getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
2657 HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
2658 RetryCounter retrier = new RetryCounter(numRetries+1, (int)pause, TimeUnit.MICROSECONDS);
2659 while(retrier.shouldRetry()) {
2660 int index = getMiniHBaseCluster().getServerWith(firstrow);
2661 if (index != -1) {
2662 return getMiniHBaseCluster().getRegionServerThreads().get(index).getRegionServer();
2663 }
2664
2665 retrier.sleepUntilNextRetry();
2666 }
2667 return null;
2668 }
2669
2670
2671
2672
2673
2674
2675
2676 public MiniMRCluster startMiniMapReduceCluster() throws IOException {
2677 startMiniMapReduceCluster(2);
2678 return mrCluster;
2679 }
2680
2681
2682
2683
2684
2685 private void forceChangeTaskLogDir() {
2686 Field logDirField;
2687 try {
2688 logDirField = TaskLog.class.getDeclaredField("LOG_DIR");
2689 logDirField.setAccessible(true);
2690
2691 Field modifiersField = Field.class.getDeclaredField("modifiers");
2692 modifiersField.setAccessible(true);
2693 modifiersField.setInt(logDirField, logDirField.getModifiers() & ~Modifier.FINAL);
2694
2695 logDirField.set(null, new File(hadoopLogDir, "userlogs"));
2696 } catch (SecurityException e) {
2697 throw new RuntimeException(e);
2698 } catch (NoSuchFieldException e) {
2699
2700 throw new RuntimeException(e);
2701 } catch (IllegalArgumentException e) {
2702 throw new RuntimeException(e);
2703 } catch (IllegalAccessException e) {
2704 throw new RuntimeException(e);
2705 }
2706 }
2707
2708
2709
2710
2711
2712
2713
2714 private void startMiniMapReduceCluster(final int servers) throws IOException {
2715 if (mrCluster != null) {
2716 throw new IllegalStateException("MiniMRCluster is already running");
2717 }
2718 LOG.info("Starting mini mapreduce cluster...");
2719 setupClusterTestDir();
2720 createDirsAndSetProperties();
2721
2722 forceChangeTaskLogDir();
2723
2724
2725
2726
2727 conf.setFloat("yarn.nodemanager.vmem-pmem-ratio", 8.0f);
2728
2729
2730
2731 conf.setBoolean("mapreduce.map.speculative", false);
2732 conf.setBoolean("mapreduce.reduce.speculative", false);
2733
2734
2735
2736 mrCluster = new MiniMRCluster(servers,
2737 FS_URI != null ? FS_URI : FileSystem.get(conf).getUri().toString(), 1,
2738 null, null, new JobConf(this.conf));
2739 JobConf jobConf = MapreduceTestingShim.getJobConf(mrCluster);
2740 if (jobConf == null) {
2741 jobConf = mrCluster.createJobConf();
2742 }
2743
2744 jobConf.set("mapreduce.cluster.local.dir",
2745 conf.get("mapreduce.cluster.local.dir"));
2746 LOG.info("Mini mapreduce cluster started");
2747
2748
2749
2750
2751 conf.set("mapreduce.jobtracker.address", jobConf.get("mapreduce.jobtracker.address"));
2752
2753 conf.set("mapreduce.framework.name", "yarn");
2754 conf.setBoolean("yarn.is.minicluster", true);
2755 String rmAddress = jobConf.get("yarn.resourcemanager.address");
2756 if (rmAddress != null) {
2757 conf.set("yarn.resourcemanager.address", rmAddress);
2758 }
2759 String historyAddress = jobConf.get("mapreduce.jobhistory.address");
2760 if (historyAddress != null) {
2761 conf.set("mapreduce.jobhistory.address", historyAddress);
2762 }
2763 String schedulerAddress =
2764 jobConf.get("yarn.resourcemanager.scheduler.address");
2765 if (schedulerAddress != null) {
2766 conf.set("yarn.resourcemanager.scheduler.address", schedulerAddress);
2767 }
2768 String mrJobHistoryWebappAddress =
2769 jobConf.get("mapreduce.jobhistory.webapp.address");
2770 if (mrJobHistoryWebappAddress != null) {
2771 conf.set("mapreduce.jobhistory.webapp.address", mrJobHistoryWebappAddress);
2772 }
2773 String yarnRMWebappAddress =
2774 jobConf.get("yarn.resourcemanager.webapp.address");
2775 if (yarnRMWebappAddress != null) {
2776 conf.set("yarn.resourcemanager.webapp.address", yarnRMWebappAddress);
2777 }
2778 }
2779
2780
2781
2782
2783 public void shutdownMiniMapReduceCluster() {
2784 if (mrCluster != null) {
2785 LOG.info("Stopping mini mapreduce cluster...");
2786 mrCluster.shutdown();
2787 mrCluster = null;
2788 LOG.info("Mini mapreduce cluster stopped");
2789 }
2790
2791 conf.set("mapreduce.jobtracker.address", "local");
2792 }
2793
2794
2795
2796
2797 public RegionServerServices createMockRegionServerService() throws IOException {
2798 return createMockRegionServerService((ServerName)null);
2799 }
2800
2801
2802
2803
2804
2805 public RegionServerServices createMockRegionServerService(RpcServerInterface rpc) throws IOException {
2806 final MockRegionServerServices rss = new MockRegionServerServices(getZooKeeperWatcher());
2807 rss.setFileSystem(getTestFileSystem());
2808 rss.setRpcServer(rpc);
2809 return rss;
2810 }
2811
2812
2813
2814
2815
2816 public RegionServerServices createMockRegionServerService(ServerName name) throws IOException {
2817 final MockRegionServerServices rss = new MockRegionServerServices(getZooKeeperWatcher(), name);
2818 rss.setFileSystem(getTestFileSystem());
2819 return rss;
2820 }
2821
2822
2823
2824
2825
2826
2827 public void enableDebug(Class<?> clazz) {
2828 Log l = LogFactory.getLog(clazz);
2829 if (l instanceof Log4JLogger) {
2830 ((Log4JLogger) l).getLogger().setLevel(org.apache.log4j.Level.DEBUG);
2831 } else if (l instanceof Jdk14Logger) {
2832 ((Jdk14Logger) l).getLogger().setLevel(java.util.logging.Level.ALL);
2833 }
2834 }
2835
2836
2837
2838
2839
2840 public void expireMasterSession() throws Exception {
2841 HMaster master = getMiniHBaseCluster().getMaster();
2842 expireSession(master.getZooKeeper(), false);
2843 }
2844
2845
2846
2847
2848
2849
2850 public void expireRegionServerSession(int index) throws Exception {
2851 HRegionServer rs = getMiniHBaseCluster().getRegionServer(index);
2852 expireSession(rs.getZooKeeper(), false);
2853 decrementMinRegionServerCount();
2854 }
2855
2856 private void decrementMinRegionServerCount() {
2857
2858
2859 decrementMinRegionServerCount(getConfiguration());
2860
2861
2862 for (MasterThread master : getHBaseCluster().getMasterThreads()) {
2863 decrementMinRegionServerCount(master.getMaster().getConfiguration());
2864 }
2865 }
2866
2867 private void decrementMinRegionServerCount(Configuration conf) {
2868 int currentCount = conf.getInt(
2869 ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1);
2870 if (currentCount != -1) {
2871 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
2872 Math.max(currentCount - 1, 1));
2873 }
2874 }
2875
2876 public void expireSession(ZooKeeperWatcher nodeZK) throws Exception {
2877 expireSession(nodeZK, false);
2878 }
2879
2880 @Deprecated
2881 public void expireSession(ZooKeeperWatcher nodeZK, Server server)
2882 throws Exception {
2883 expireSession(nodeZK, false);
2884 }
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897 public void expireSession(ZooKeeperWatcher nodeZK, boolean checkStatus)
2898 throws Exception {
2899 Configuration c = new Configuration(this.conf);
2900 String quorumServers = ZKConfig.getZKQuorumServersString(c);
2901 ZooKeeper zk = nodeZK.getRecoverableZooKeeper().getZooKeeper();
2902 byte[] password = zk.getSessionPasswd();
2903 long sessionID = zk.getSessionId();
2904
2905
2906
2907
2908
2909
2910
2911
2912 ZooKeeper monitor = new ZooKeeper(quorumServers,
2913 1000, new org.apache.zookeeper.Watcher(){
2914 @Override
2915 public void process(WatchedEvent watchedEvent) {
2916 LOG.info("Monitor ZKW received event="+watchedEvent);
2917 }
2918 } , sessionID, password);
2919
2920
2921 ZooKeeper newZK = new ZooKeeper(quorumServers,
2922 1000, EmptyWatcher.instance, sessionID, password);
2923
2924
2925
2926 long start = System.currentTimeMillis();
2927 while (newZK.getState() != States.CONNECTED
2928 && System.currentTimeMillis() - start < 1000) {
2929 Thread.sleep(1);
2930 }
2931 newZK.close();
2932 LOG.info("ZK Closed Session 0x" + Long.toHexString(sessionID));
2933
2934
2935 monitor.close();
2936
2937 if (checkStatus) {
2938 new HTable(new Configuration(conf), TableName.META_TABLE_NAME).close();
2939 }
2940 }
2941
2942
2943
2944
2945
2946
2947
2948 public MiniHBaseCluster getHBaseCluster() {
2949 return getMiniHBaseCluster();
2950 }
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960 public HBaseCluster getHBaseClusterInterface() {
2961
2962
2963 return hbaseCluster;
2964 }
2965
2966
2967
2968
2969
2970
2971
2972 public Connection getConnection() throws IOException {
2973 if (this.connection == null) {
2974 this.connection = ConnectionFactory.createConnection(this.conf);
2975 }
2976 return this.connection;
2977 }
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988 public synchronized HBaseAdmin getHBaseAdmin()
2989 throws IOException {
2990 if (hbaseAdmin == null){
2991 this.hbaseAdmin = new HBaseAdminForTests(getConnection());
2992 }
2993 return hbaseAdmin;
2994 }
2995
2996 private HBaseAdminForTests hbaseAdmin = null;
2997 private static class HBaseAdminForTests extends HBaseAdmin {
2998 public HBaseAdminForTests(Connection connection) throws MasterNotRunningException,
2999 ZooKeeperConnectionException, IOException {
3000 super(connection);
3001 }
3002
3003 @Override
3004 public synchronized void close() throws IOException {
3005 LOG.warn("close() called on HBaseAdmin instance returned from " +
3006 "HBaseTestingUtility.getHBaseAdmin()");
3007 }
3008
3009 private synchronized void close0() throws IOException {
3010 super.close();
3011 }
3012 }
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023 public synchronized ZooKeeperWatcher getZooKeeperWatcher()
3024 throws IOException {
3025 if (zooKeeperWatcher == null) {
3026 zooKeeperWatcher = new ZooKeeperWatcher(conf, "testing utility",
3027 new Abortable() {
3028 @Override public void abort(String why, Throwable e) {
3029 throw new RuntimeException("Unexpected abort in HBaseTestingUtility:"+why, e);
3030 }
3031 @Override public boolean isAborted() {return false;}
3032 });
3033 }
3034 return zooKeeperWatcher;
3035 }
3036 private ZooKeeperWatcher zooKeeperWatcher;
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046 public void closeRegion(String regionName) throws IOException {
3047 closeRegion(Bytes.toBytes(regionName));
3048 }
3049
3050
3051
3052
3053
3054
3055
3056 public void closeRegion(byte[] regionName) throws IOException {
3057 getHBaseAdmin().closeRegion(regionName, null);
3058 }
3059
3060
3061
3062
3063
3064
3065
3066
3067 public void closeRegionByRow(String row, RegionLocator table) throws IOException {
3068 closeRegionByRow(Bytes.toBytes(row), table);
3069 }
3070
3071
3072
3073
3074
3075
3076
3077
3078 public void closeRegionByRow(byte[] row, RegionLocator table) throws IOException {
3079 HRegionLocation hrl = table.getRegionLocation(row);
3080 closeRegion(hrl.getRegionInfo().getRegionName());
3081 }
3082
3083
3084
3085
3086
3087
3088
3089
3090 public HRegion getSplittableRegion(TableName tableName, int maxAttempts) {
3091 List<HRegion> regions = getHBaseCluster().getRegions(tableName);
3092 int regCount = regions.size();
3093 Set<Integer> attempted = new HashSet<Integer>();
3094 int idx;
3095 int attempts = 0;
3096 do {
3097 regions = getHBaseCluster().getRegions(tableName);
3098 if (regCount != regions.size()) {
3099
3100 attempted.clear();
3101 }
3102 regCount = regions.size();
3103
3104
3105 if (regCount > 0) {
3106 idx = random.nextInt(regCount);
3107
3108 if (attempted.contains(idx))
3109 continue;
3110 try {
3111 regions.get(idx).checkSplit();
3112 return regions.get(idx);
3113 } catch (Exception ex) {
3114 LOG.warn("Caught exception", ex);
3115 attempted.add(idx);
3116 }
3117 }
3118 attempts++;
3119 } while (maxAttempts == -1 || attempts < maxAttempts);
3120 return null;
3121 }
3122
3123 public MiniZooKeeperCluster getZkCluster() {
3124 return zkCluster;
3125 }
3126
3127 public void setZkCluster(MiniZooKeeperCluster zkCluster) {
3128 this.passedZkCluster = true;
3129 this.zkCluster = zkCluster;
3130 conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkCluster.getClientPort());
3131 }
3132
3133 public MiniDFSCluster getDFSCluster() {
3134 return dfsCluster;
3135 }
3136
3137 public void setDFSCluster(MiniDFSCluster cluster) throws IllegalStateException, IOException {
3138 setDFSCluster(cluster, true);
3139 }
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149 public void setDFSCluster(MiniDFSCluster cluster, boolean requireDown)
3150 throws IllegalStateException, IOException {
3151 if (dfsCluster != null && requireDown && dfsCluster.isClusterUp()) {
3152 throw new IllegalStateException("DFSCluster is already running! Shut it down first.");
3153 }
3154 this.dfsCluster = cluster;
3155 this.setFs();
3156 }
3157
3158 public FileSystem getTestFileSystem() throws IOException {
3159 return HFileSystem.get(conf);
3160 }
3161
3162
3163
3164
3165
3166
3167
3168
3169 public void waitTableAvailable(TableName table)
3170 throws InterruptedException, IOException {
3171 waitTableAvailable(table.getName(), 30000);
3172 }
3173
3174 public void waitTableAvailable(TableName table, long timeoutMillis)
3175 throws InterruptedException, IOException {
3176 waitFor(timeoutMillis, predicateTableAvailable(table));
3177 }
3178
3179 public String explainTableAvailability(TableName tableName) throws IOException {
3180 String msg = explainTableState(tableName) + ",";
3181 if (getHBaseCluster().getMaster().isAlive()) {
3182 Map<HRegionInfo, ServerName> assignments =
3183 getHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
3184 .getRegionAssignments();
3185 final List<Pair<HRegionInfo, ServerName>> metaLocations =
3186 MetaTableAccessor
3187 .getTableRegionsAndLocations(getZooKeeperWatcher(), connection, tableName);
3188 for (Pair<HRegionInfo, ServerName> metaLocation : metaLocations) {
3189 HRegionInfo hri = metaLocation.getFirst();
3190 ServerName sn = metaLocation.getSecond();
3191 if (!assignments.containsKey(hri)) {
3192 msg += ", region " + hri
3193 + " not assigned, but found in meta, it expected to be on " + sn;
3194
3195 } else if (sn == null) {
3196 msg += ", region " + hri
3197 + " assigned, but has no server in meta";
3198 } else if (!sn.equals(assignments.get(hri))) {
3199 msg += ", region " + hri
3200 + " assigned, but has different servers in meta and AM ( " +
3201 sn + " <> " + assignments.get(hri);
3202 }
3203 }
3204 }
3205 return msg;
3206 }
3207
3208 public String explainTableState(TableName tableName) throws IOException {
3209 try {
3210 if (getHBaseAdmin().isTableEnabled(tableName))
3211 return "table enabled in zk";
3212 else if (getHBaseAdmin().isTableDisabled(tableName))
3213 return "table disabled in zk";
3214 else
3215 return "table in uknown state";
3216 } catch (TableNotFoundException e) {
3217 return "table not exists";
3218 }
3219 }
3220
3221
3222
3223
3224
3225
3226
3227
3228 public void waitTableAvailable(byte[] table, long timeoutMillis)
3229 throws InterruptedException, IOException {
3230 waitTableAvailable(getHBaseAdmin(), table, timeoutMillis);
3231 }
3232
3233 public void waitTableAvailable(Admin admin, byte[] table, long timeoutMillis)
3234 throws InterruptedException, IOException {
3235 long startWait = System.currentTimeMillis();
3236 while (!admin.isTableAvailable(TableName.valueOf(table))) {
3237 assertTrue("Timed out waiting for table to become available " +
3238 Bytes.toStringBinary(table),
3239 System.currentTimeMillis() - startWait < timeoutMillis);
3240 Thread.sleep(200);
3241 }
3242 }
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253 public void waitTableEnabled(TableName table)
3254 throws InterruptedException, IOException {
3255 waitTableEnabled(table, 30000);
3256 }
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267 public void waitTableEnabled(byte[] table, long timeoutMillis)
3268 throws InterruptedException, IOException {
3269 waitTableEnabled(TableName.valueOf(table), timeoutMillis);
3270 }
3271
3272 public void waitTableEnabled(TableName table, long timeoutMillis)
3273 throws IOException {
3274 waitFor(timeoutMillis, predicateTableEnabled(table));
3275 }
3276
3277
3278
3279
3280
3281
3282
3283
3284 public void waitTableDisabled(byte[] table)
3285 throws InterruptedException, IOException {
3286 waitTableDisabled(getHBaseAdmin(), table, 30000);
3287 }
3288
3289 public void waitTableDisabled(Admin admin, byte[] table)
3290 throws InterruptedException, IOException {
3291 waitTableDisabled(admin, table, 30000);
3292 }
3293
3294
3295
3296
3297
3298
3299
3300
3301 public void waitTableDisabled(byte[] table, long timeoutMillis)
3302 throws InterruptedException, IOException {
3303 waitTableDisabled(getHBaseAdmin(), table, timeoutMillis);
3304 }
3305
3306 public void waitTableDisabled(Admin admin, byte[] table, long timeoutMillis)
3307 throws InterruptedException, IOException {
3308 TableName tableName = TableName.valueOf(table);
3309 long startWait = System.currentTimeMillis();
3310 while (!admin.isTableDisabled(tableName)) {
3311 assertTrue("Timed out waiting for table to become disabled " +
3312 Bytes.toStringBinary(table),
3313 System.currentTimeMillis() - startWait < timeoutMillis);
3314 Thread.sleep(200);
3315 }
3316 }
3317
3318
3319
3320
3321
3322
3323
3324
3325 public boolean ensureSomeRegionServersAvailable(final int num)
3326 throws IOException {
3327 boolean startedServer = false;
3328 MiniHBaseCluster hbaseCluster = getMiniHBaseCluster();
3329 for (int i=hbaseCluster.getLiveRegionServerThreads().size(); i<num; ++i) {
3330 LOG.info("Started new server=" + hbaseCluster.startRegionServer());
3331 startedServer = true;
3332 }
3333
3334 return startedServer;
3335 }
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346 public boolean ensureSomeNonStoppedRegionServersAvailable(final int num)
3347 throws IOException {
3348 boolean startedServer = ensureSomeRegionServersAvailable(num);
3349
3350 int nonStoppedServers = 0;
3351 for (JVMClusterUtil.RegionServerThread rst :
3352 getMiniHBaseCluster().getRegionServerThreads()) {
3353
3354 HRegionServer hrs = rst.getRegionServer();
3355 if (hrs.isStopping() || hrs.isStopped()) {
3356 LOG.info("A region server is stopped or stopping:"+hrs);
3357 } else {
3358 nonStoppedServers++;
3359 }
3360 }
3361 for (int i=nonStoppedServers; i<num; ++i) {
3362 LOG.info("Started new server=" + getMiniHBaseCluster().startRegionServer());
3363 startedServer = true;
3364 }
3365 return startedServer;
3366 }
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378 public static User getDifferentUser(final Configuration c,
3379 final String differentiatingSuffix)
3380 throws IOException {
3381 FileSystem currentfs = FileSystem.get(c);
3382 if (!(currentfs instanceof DistributedFileSystem) || User.isHBaseSecurityEnabled(c)) {
3383 return User.getCurrent();
3384 }
3385
3386
3387 String username = User.getCurrent().getName() +
3388 differentiatingSuffix;
3389 User user = User.createUserForTesting(c, username,
3390 new String[]{"supergroup"});
3391 return user;
3392 }
3393
3394 public static NavigableSet<String> getAllOnlineRegions(MiniHBaseCluster cluster)
3395 throws IOException {
3396 NavigableSet<String> online = new TreeSet<String>();
3397 for (RegionServerThread rst : cluster.getLiveRegionServerThreads()) {
3398 try {
3399 for (HRegionInfo region :
3400 ProtobufUtil.getOnlineRegions(rst.getRegionServer().getRSRpcServices())) {
3401 online.add(region.getRegionNameAsString());
3402 }
3403 } catch (RegionServerStoppedException e) {
3404
3405 }
3406 }
3407 for (MasterThread mt : cluster.getLiveMasterThreads()) {
3408 try {
3409 for (HRegionInfo region :
3410 ProtobufUtil.getOnlineRegions(mt.getMaster().getRSRpcServices())) {
3411 online.add(region.getRegionNameAsString());
3412 }
3413 } catch (RegionServerStoppedException e) {
3414
3415 } catch (ServerNotRunningYetException e) {
3416
3417 }
3418 }
3419 return online;
3420 }
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435 public static void setMaxRecoveryErrorCount(final OutputStream stream,
3436 final int max) {
3437 try {
3438 Class<?> [] clazzes = DFSClient.class.getDeclaredClasses();
3439 for (Class<?> clazz: clazzes) {
3440 String className = clazz.getSimpleName();
3441 if (className.equals("DFSOutputStream")) {
3442 if (clazz.isInstance(stream)) {
3443 Field maxRecoveryErrorCountField =
3444 stream.getClass().getDeclaredField("maxRecoveryErrorCount");
3445 maxRecoveryErrorCountField.setAccessible(true);
3446 maxRecoveryErrorCountField.setInt(stream, max);
3447 break;
3448 }
3449 }
3450 }
3451 } catch (Exception e) {
3452 LOG.info("Could not set max recovery field", e);
3453 }
3454 }
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464 public boolean assignRegion(final HRegionInfo regionInfo)
3465 throws IOException, InterruptedException {
3466 final AssignmentManager am = getHBaseCluster().getMaster().getAssignmentManager();
3467 am.assign(regionInfo, true);
3468 return am.waitForAssignment(regionInfo);
3469 }
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480 public void waitUntilAllRegionsAssigned(final TableName tableName) throws IOException {
3481 waitUntilAllRegionsAssigned(
3482 tableName,
3483 this.conf.getLong("hbase.client.sync.wait.timeout.msec", 60000));
3484 }
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495 public void waitUntilAllRegionsAssigned(final TableName tableName, final long timeout)
3496 throws IOException {
3497 final Table meta = new HTable(getConfiguration(), TableName.META_TABLE_NAME);
3498 try {
3499 waitFor(timeout, 200, true, new Predicate<IOException>() {
3500 @Override
3501 public boolean evaluate() throws IOException {
3502 boolean allRegionsAssigned = true;
3503 Scan scan = new Scan();
3504 scan.addFamily(HConstants.CATALOG_FAMILY);
3505 ResultScanner s = meta.getScanner(scan);
3506 try {
3507 Result r;
3508 while ((r = s.next()) != null) {
3509 byte[] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
3510 HRegionInfo info = HRegionInfo.parseFromOrNull(b);
3511 if (info != null && info.getTable().equals(tableName)) {
3512 b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
3513 allRegionsAssigned &= (b != null);
3514 }
3515 }
3516 } finally {
3517 s.close();
3518 }
3519 return allRegionsAssigned;
3520 }
3521 });
3522 } finally {
3523 meta.close();
3524 }
3525
3526
3527 if (!getHBaseClusterInterface().isDistributedCluster()) {
3528
3529
3530 HMaster master = getHBaseCluster().getMaster();
3531 final RegionStates states = master.getAssignmentManager().getRegionStates();
3532 waitFor(timeout, 200, new ExplainingPredicate<IOException>() {
3533 @Override
3534 public String explainFailure() throws IOException {
3535 return explainTableAvailability(tableName);
3536 }
3537
3538 @Override
3539 public boolean evaluate() throws IOException {
3540 List<HRegionInfo> hris = states.getRegionsOfTable(tableName);
3541 return hris != null && !hris.isEmpty();
3542 }
3543 });
3544 }
3545 }
3546
3547
3548
3549
3550
3551 public static List<Cell> getFromStoreFile(HStore store,
3552 Get get) throws IOException {
3553 Scan scan = new Scan(get);
3554 InternalScanner scanner = (InternalScanner) store.getScanner(scan,
3555 scan.getFamilyMap().get(store.getFamily().getName()),
3556
3557
3558 0);
3559
3560 List<Cell> result = new ArrayList<Cell>();
3561 scanner.next(result);
3562 if (!result.isEmpty()) {
3563
3564 Cell kv = result.get(0);
3565 if (!CellUtil.matchingRow(kv, get.getRow())) {
3566 result.clear();
3567 }
3568 }
3569 scanner.close();
3570 return result;
3571 }
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581 public byte[][] getRegionSplitStartKeys(byte[] startKey, byte[] endKey, int numRegions){
3582 assertTrue(numRegions>3);
3583 byte [][] tmpSplitKeys = Bytes.split(startKey, endKey, numRegions - 3);
3584 byte [][] result = new byte[tmpSplitKeys.length+1][];
3585 System.arraycopy(tmpSplitKeys, 0, result, 1, tmpSplitKeys.length);
3586 result[0] = HConstants.EMPTY_BYTE_ARRAY;
3587 return result;
3588 }
3589
3590
3591
3592
3593
3594 public static List<Cell> getFromStoreFile(HStore store,
3595 byte [] row,
3596 NavigableSet<byte[]> columns
3597 ) throws IOException {
3598 Get get = new Get(row);
3599 Map<byte[], NavigableSet<byte[]>> s = get.getFamilyMap();
3600 s.put(store.getFamily().getName(), columns);
3601
3602 return getFromStoreFile(store,get);
3603 }
3604
3605
3606
3607
3608
3609 public static ZooKeeperWatcher getZooKeeperWatcher(
3610 HBaseTestingUtility TEST_UTIL) throws ZooKeeperConnectionException,
3611 IOException {
3612 ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
3613 "unittest", new Abortable() {
3614 boolean aborted = false;
3615
3616 @Override
3617 public void abort(String why, Throwable e) {
3618 aborted = true;
3619 throw new RuntimeException("Fatal ZK error, why=" + why, e);
3620 }
3621
3622 @Override
3623 public boolean isAborted() {
3624 return aborted;
3625 }
3626 });
3627 return zkw;
3628 }
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641 public static ZooKeeperWatcher createAndForceNodeToOpenedState(
3642 HBaseTestingUtility TEST_UTIL, Region region,
3643 ServerName serverName) throws ZooKeeperConnectionException,
3644 IOException, KeeperException, NodeExistsException {
3645 return createAndForceNodeToOpenedState(TEST_UTIL, (HRegion)region, serverName);
3646 }
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659 public static ZooKeeperWatcher createAndForceNodeToOpenedState(
3660 HBaseTestingUtility TEST_UTIL, HRegion region,
3661 ServerName serverName) throws ZooKeeperConnectionException,
3662 IOException, KeeperException, NodeExistsException {
3663 ZooKeeperWatcher zkw = getZooKeeperWatcher(TEST_UTIL);
3664 ZKAssign.createNodeOffline(zkw, region.getRegionInfo(), serverName);
3665 int version = ZKAssign.transitionNodeOpening(zkw, region
3666 .getRegionInfo(), serverName);
3667 ZKAssign.transitionNodeOpened(zkw, region.getRegionInfo(), serverName,
3668 version);
3669 return zkw;
3670 }
3671
3672 public static void assertKVListsEqual(String additionalMsg,
3673 final List<? extends Cell> expected,
3674 final List<? extends Cell> actual) {
3675 final int eLen = expected.size();
3676 final int aLen = actual.size();
3677 final int minLen = Math.min(eLen, aLen);
3678
3679 int i;
3680 for (i = 0; i < minLen
3681 && KeyValue.COMPARATOR.compare(expected.get(i), actual.get(i)) == 0;
3682 ++i) {}
3683
3684 if (additionalMsg == null) {
3685 additionalMsg = "";
3686 }
3687 if (!additionalMsg.isEmpty()) {
3688 additionalMsg = ". " + additionalMsg;
3689 }
3690
3691 if (eLen != aLen || i != minLen) {
3692 throw new AssertionError(
3693 "Expected and actual KV arrays differ at position " + i + ": " +
3694 safeGetAsStr(expected, i) + " (length " + eLen +") vs. " +
3695 safeGetAsStr(actual, i) + " (length " + aLen + ")" + additionalMsg);
3696 }
3697 }
3698
3699 public static <T> String safeGetAsStr(List<T> lst, int i) {
3700 if (0 <= i && i < lst.size()) {
3701 return lst.get(i).toString();
3702 } else {
3703 return "<out_of_range>";
3704 }
3705 }
3706
3707 public String getClusterKey() {
3708 return conf.get(HConstants.ZOOKEEPER_QUORUM) + ":"
3709 + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT) + ":"
3710 + conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
3711 HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
3712 }
3713
3714
3715 public HTable createRandomTable(String tableName,
3716 final Collection<String> families,
3717 final int maxVersions,
3718 final int numColsPerRow,
3719 final int numFlushes,
3720 final int numRegions,
3721 final int numRowsPerFlush)
3722 throws IOException, InterruptedException {
3723
3724 LOG.info("\n\nCreating random table " + tableName + " with " + numRegions +
3725 " regions, " + numFlushes + " storefiles per region, " +
3726 numRowsPerFlush + " rows per flush, maxVersions=" + maxVersions +
3727 "\n");
3728
3729 final Random rand = new Random(tableName.hashCode() * 17L + 12938197137L);
3730 final int numCF = families.size();
3731 final byte[][] cfBytes = new byte[numCF][];
3732 {
3733 int cfIndex = 0;
3734 for (String cf : families) {
3735 cfBytes[cfIndex++] = Bytes.toBytes(cf);
3736 }
3737 }
3738
3739 final int actualStartKey = 0;
3740 final int actualEndKey = Integer.MAX_VALUE;
3741 final int keysPerRegion = (actualEndKey - actualStartKey) / numRegions;
3742 final int splitStartKey = actualStartKey + keysPerRegion;
3743 final int splitEndKey = actualEndKey - keysPerRegion;
3744 final String keyFormat = "%08x";
3745 final HTable table = createTable(tableName, cfBytes,
3746 maxVersions,
3747 Bytes.toBytes(String.format(keyFormat, splitStartKey)),
3748 Bytes.toBytes(String.format(keyFormat, splitEndKey)),
3749 numRegions);
3750
3751 if (hbaseCluster != null) {
3752 getMiniHBaseCluster().flushcache(TableName.META_TABLE_NAME);
3753 }
3754
3755 for (int iFlush = 0; iFlush < numFlushes; ++iFlush) {
3756 for (int iRow = 0; iRow < numRowsPerFlush; ++iRow) {
3757 final byte[] row = Bytes.toBytes(String.format(keyFormat,
3758 actualStartKey + rand.nextInt(actualEndKey - actualStartKey)));
3759
3760 Put put = new Put(row);
3761 Delete del = new Delete(row);
3762 for (int iCol = 0; iCol < numColsPerRow; ++iCol) {
3763 final byte[] cf = cfBytes[rand.nextInt(numCF)];
3764 final long ts = rand.nextInt();
3765 final byte[] qual = Bytes.toBytes("col" + iCol);
3766 if (rand.nextBoolean()) {
3767 final byte[] value = Bytes.toBytes("value_for_row_" + iRow +
3768 "_cf_" + Bytes.toStringBinary(cf) + "_col_" + iCol + "_ts_" +
3769 ts + "_random_" + rand.nextLong());
3770 put.add(cf, qual, ts, value);
3771 } else if (rand.nextDouble() < 0.8) {
3772 del.deleteColumn(cf, qual, ts);
3773 } else {
3774 del.deleteColumns(cf, qual, ts);
3775 }
3776 }
3777
3778 if (!put.isEmpty()) {
3779 table.put(put);
3780 }
3781
3782 if (!del.isEmpty()) {
3783 table.delete(del);
3784 }
3785 }
3786 LOG.info("Initiating flush #" + iFlush + " for table " + tableName);
3787 table.flushCommits();
3788 if (hbaseCluster != null) {
3789 getMiniHBaseCluster().flushcache(table.getName());
3790 }
3791 }
3792
3793 return table;
3794 }
3795
3796 private static final int MIN_RANDOM_PORT = 0xc000;
3797 private static final int MAX_RANDOM_PORT = 0xfffe;
3798 private static Random random = new Random();
3799
3800
3801
3802
3803
3804 public static int randomPort() {
3805 return MIN_RANDOM_PORT
3806 + random.nextInt(MAX_RANDOM_PORT - MIN_RANDOM_PORT);
3807 }
3808
3809
3810
3811
3812
3813 public static int randomFreePort() {
3814 int port = 0;
3815 do {
3816 port = randomPort();
3817 if (takenRandomPorts.contains(port)) {
3818 port = 0;
3819 continue;
3820 }
3821 takenRandomPorts.add(port);
3822
3823 try {
3824 ServerSocket sock = new ServerSocket(port);
3825 sock.close();
3826 } catch (IOException ex) {
3827 port = 0;
3828 }
3829 } while (port == 0);
3830 return port;
3831 }
3832
3833
3834 public static String randomMultiCastAddress() {
3835 return "226.1.1." + random.nextInt(254);
3836 }
3837
3838
3839
3840 public static void waitForHostPort(String host, int port)
3841 throws IOException {
3842 final int maxTimeMs = 10000;
3843 final int maxNumAttempts = maxTimeMs / HConstants.SOCKET_RETRY_WAIT_MS;
3844 IOException savedException = null;
3845 LOG.info("Waiting for server at " + host + ":" + port);
3846 for (int attempt = 0; attempt < maxNumAttempts; ++attempt) {
3847 try {
3848 Socket sock = new Socket(InetAddress.getByName(host), port);
3849 sock.close();
3850 savedException = null;
3851 LOG.info("Server at " + host + ":" + port + " is available");
3852 break;
3853 } catch (UnknownHostException e) {
3854 throw new IOException("Failed to look up " + host, e);
3855 } catch (IOException e) {
3856 savedException = e;
3857 }
3858 Threads.sleepWithoutInterrupt(HConstants.SOCKET_RETRY_WAIT_MS);
3859 }
3860
3861 if (savedException != null) {
3862 throw savedException;
3863 }
3864 }
3865
3866
3867
3868
3869
3870
3871 public static int createPreSplitLoadTestTable(Configuration conf,
3872 TableName tableName, byte[] columnFamily, Algorithm compression,
3873 DataBlockEncoding dataBlockEncoding) throws IOException {
3874 return createPreSplitLoadTestTable(conf, tableName,
3875 columnFamily, compression, dataBlockEncoding, DEFAULT_REGIONS_PER_SERVER, 1,
3876 Durability.USE_DEFAULT);
3877 }
3878
3879
3880
3881
3882
3883 public static int createPreSplitLoadTestTable(Configuration conf,
3884 TableName tableName, byte[] columnFamily, Algorithm compression,
3885 DataBlockEncoding dataBlockEncoding, int numRegionsPerServer, int regionReplication,
3886 Durability durability)
3887 throws IOException {
3888 HTableDescriptor desc = new HTableDescriptor(tableName);
3889 desc.setDurability(durability);
3890 desc.setRegionReplication(regionReplication);
3891 HColumnDescriptor hcd = new HColumnDescriptor(columnFamily);
3892 hcd.setDataBlockEncoding(dataBlockEncoding);
3893 hcd.setCompressionType(compression);
3894 return createPreSplitLoadTestTable(conf, desc, hcd, numRegionsPerServer);
3895 }
3896
3897
3898
3899
3900
3901
3902 public static int createPreSplitLoadTestTable(Configuration conf,
3903 TableName tableName, byte[][] columnFamilies, Algorithm compression,
3904 DataBlockEncoding dataBlockEncoding, int numRegionsPerServer, int regionReplication,
3905 Durability durability)
3906 throws IOException {
3907 HTableDescriptor desc = new HTableDescriptor(tableName);
3908 desc.setDurability(durability);
3909 desc.setRegionReplication(regionReplication);
3910 HColumnDescriptor[] hcds = new HColumnDescriptor[columnFamilies.length];
3911 for (int i = 0; i < columnFamilies.length; i++) {
3912 HColumnDescriptor hcd = new HColumnDescriptor(columnFamilies[i]);
3913 hcd.setDataBlockEncoding(dataBlockEncoding);
3914 hcd.setCompressionType(compression);
3915 hcds[i] = hcd;
3916 }
3917 return createPreSplitLoadTestTable(conf, desc, hcds, numRegionsPerServer);
3918 }
3919
3920
3921
3922
3923
3924
3925 public static int createPreSplitLoadTestTable(Configuration conf,
3926 HTableDescriptor desc, HColumnDescriptor hcd) throws IOException {
3927 return createPreSplitLoadTestTable(conf, desc, hcd, DEFAULT_REGIONS_PER_SERVER);
3928 }
3929
3930
3931
3932
3933
3934
3935 public static int createPreSplitLoadTestTable(Configuration conf,
3936 HTableDescriptor desc, HColumnDescriptor hcd, int numRegionsPerServer) throws IOException {
3937 return createPreSplitLoadTestTable(conf, desc, new HColumnDescriptor[] {hcd},
3938 numRegionsPerServer);
3939 }
3940
3941
3942
3943
3944
3945
3946 public static int createPreSplitLoadTestTable(Configuration conf,
3947 HTableDescriptor desc, HColumnDescriptor[] hcds, int numRegionsPerServer) throws IOException {
3948 for (HColumnDescriptor hcd : hcds) {
3949 if (!desc.hasFamily(hcd.getName())) {
3950 desc.addFamily(hcd);
3951 }
3952 }
3953
3954 int totalNumberOfRegions = 0;
3955 Connection unmanagedConnection = ConnectionFactory.createConnection(conf);
3956 Admin admin = unmanagedConnection.getAdmin();
3957
3958 try {
3959
3960
3961
3962 int numberOfServers = admin.getClusterStatus().getServers().size();
3963 if (numberOfServers == 0) {
3964 throw new IllegalStateException("No live regionservers");
3965 }
3966
3967 totalNumberOfRegions = numberOfServers * numRegionsPerServer;
3968 LOG.info("Number of live regionservers: " + numberOfServers + ", " +
3969 "pre-splitting table into " + totalNumberOfRegions + " regions " +
3970 "(regions per server: " + numRegionsPerServer + ")");
3971
3972 byte[][] splits = new RegionSplitter.HexStringSplit().split(
3973 totalNumberOfRegions);
3974
3975 admin.createTable(desc, splits);
3976 } catch (MasterNotRunningException e) {
3977 LOG.error("Master not running", e);
3978 throw new IOException(e);
3979 } catch (TableExistsException e) {
3980 LOG.warn("Table " + desc.getTableName() +
3981 " already exists, continuing");
3982 } finally {
3983 admin.close();
3984 unmanagedConnection.close();
3985 }
3986 return totalNumberOfRegions;
3987 }
3988
3989 public static int getMetaRSPort(Configuration conf) throws IOException {
3990 try (Connection c = ConnectionFactory.createConnection();
3991 RegionLocator locator = c.getRegionLocator(TableName.META_TABLE_NAME)) {
3992 return locator.getRegionLocation(Bytes.toBytes("")).getPort();
3993 }
3994 }
3995
3996
3997
3998
3999
4000
4001
4002 public void assertRegionOnServer(
4003 final HRegionInfo hri, final ServerName server,
4004 final long timeout) throws IOException, InterruptedException {
4005 long timeoutTime = System.currentTimeMillis() + timeout;
4006 while (true) {
4007 List<HRegionInfo> regions = getHBaseAdmin().getOnlineRegions(server);
4008 if (regions.contains(hri)) return;
4009 long now = System.currentTimeMillis();
4010 if (now > timeoutTime) break;
4011 Thread.sleep(10);
4012 }
4013 fail("Could not find region " + hri.getRegionNameAsString()
4014 + " on server " + server);
4015 }
4016
4017
4018
4019
4020
4021 public void assertRegionOnlyOnServer(
4022 final HRegionInfo hri, final ServerName server,
4023 final long timeout) throws IOException, InterruptedException {
4024 long timeoutTime = System.currentTimeMillis() + timeout;
4025 while (true) {
4026 List<HRegionInfo> regions = getHBaseAdmin().getOnlineRegions(server);
4027 if (regions.contains(hri)) {
4028 List<JVMClusterUtil.RegionServerThread> rsThreads =
4029 getHBaseCluster().getLiveRegionServerThreads();
4030 for (JVMClusterUtil.RegionServerThread rsThread: rsThreads) {
4031 HRegionServer rs = rsThread.getRegionServer();
4032 if (server.equals(rs.getServerName())) {
4033 continue;
4034 }
4035 Collection<Region> hrs = rs.getOnlineRegionsLocalContext();
4036 for (Region r: hrs) {
4037 assertTrue("Region should not be double assigned",
4038 r.getRegionInfo().getRegionId() != hri.getRegionId());
4039 }
4040 }
4041 return;
4042 }
4043 long now = System.currentTimeMillis();
4044 if (now > timeoutTime) break;
4045 Thread.sleep(10);
4046 }
4047 fail("Could not find region " + hri.getRegionNameAsString()
4048 + " on server " + server);
4049 }
4050
4051 public HRegion createTestRegion(String tableName, HColumnDescriptor hcd)
4052 throws IOException {
4053 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
4054 htd.addFamily(hcd);
4055 HRegionInfo info =
4056 new HRegionInfo(TableName.valueOf(tableName), null, null, false);
4057 HRegion region =
4058 HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), htd);
4059 return region;
4060 }
4061
4062 public void setFileSystemURI(String fsURI) {
4063 FS_URI = fsURI;
4064 }
4065
4066
4067
4068
4069 public <E extends Exception> long waitFor(long timeout, Predicate<E> predicate)
4070 throws E {
4071 return Waiter.waitFor(this.conf, timeout, predicate);
4072 }
4073
4074
4075
4076
4077 public <E extends Exception> long waitFor(long timeout, long interval, Predicate<E> predicate)
4078 throws E {
4079 return Waiter.waitFor(this.conf, timeout, interval, predicate);
4080 }
4081
4082
4083
4084
4085 public <E extends Exception> long waitFor(long timeout, long interval,
4086 boolean failIfTimeout, Predicate<E> predicate) throws E {
4087 return Waiter.waitFor(this.conf, timeout, interval, failIfTimeout, predicate);
4088 }
4089
4090
4091
4092
4093 public ExplainingPredicate<IOException> predicateNoRegionsInTransition() {
4094 return new ExplainingPredicate<IOException>() {
4095 @Override
4096 public String explainFailure() throws IOException {
4097 final RegionStates regionStates = getMiniHBaseCluster().getMaster()
4098 .getAssignmentManager().getRegionStates();
4099 return "found in transition: " + regionStates.getRegionsInTransition().toString();
4100 }
4101
4102 @Override
4103 public boolean evaluate() throws IOException {
4104 HMaster master = getMiniHBaseCluster().getMaster();
4105 if (master == null) return false;
4106 AssignmentManager am = master.getAssignmentManager();
4107 if (am == null) return false;
4108 final RegionStates regionStates = am.getRegionStates();
4109 return !regionStates.isRegionsInTransition();
4110 }
4111 };
4112 }
4113
4114
4115
4116
4117 public Waiter.Predicate<IOException> predicateTableEnabled(final TableName tableName) {
4118 return new ExplainingPredicate<IOException>() {
4119 @Override
4120 public String explainFailure() throws IOException {
4121 return explainTableState(tableName);
4122 }
4123
4124 @Override
4125 public boolean evaluate() throws IOException {
4126 return getHBaseAdmin().tableExists(tableName) && getHBaseAdmin().isTableEnabled(tableName);
4127 }
4128 };
4129 }
4130
4131
4132
4133
4134 public Waiter.Predicate<IOException> predicateTableDisabled(final TableName tableName) {
4135 return new ExplainingPredicate<IOException>() {
4136 @Override
4137 public String explainFailure() throws IOException {
4138 return explainTableState(tableName);
4139 }
4140
4141 @Override
4142 public boolean evaluate() throws IOException {
4143 return getHBaseAdmin().isTableDisabled(tableName);
4144 }
4145 };
4146 }
4147
4148
4149
4150
4151 public Waiter.Predicate<IOException> predicateTableAvailable(final TableName tableName) {
4152 return new ExplainingPredicate<IOException>() {
4153 @Override
4154 public String explainFailure() throws IOException {
4155 return explainTableAvailability(tableName);
4156 }
4157
4158 @Override
4159 public boolean evaluate() throws IOException {
4160 boolean tableAvailable = getHBaseAdmin().isTableAvailable(tableName);
4161 if (tableAvailable) {
4162 try (Table table = getConnection().getTable(tableName)) {
4163 HTableDescriptor htd = table.getTableDescriptor();
4164 for (HRegionLocation loc : getConnection().getRegionLocator(tableName)
4165 .getAllRegionLocations()) {
4166 Scan scan = new Scan();
4167 scan.setStartRow(loc.getRegionInfo().getStartKey());
4168 scan.setStopRow(loc.getRegionInfo().getEndKey());
4169 scan.setMaxResultsPerColumnFamily(1);
4170 scan.setCaching(1);
4171 scan.setCacheBlocks(false);
4172 for (byte[] family : htd.getFamiliesKeys()) {
4173 scan.addFamily(family);
4174 }
4175 try (ResultScanner scanner = table.getScanner(scan)) {
4176 scanner.next();
4177 }
4178 }
4179 }
4180 }
4181 return tableAvailable;
4182 }
4183 };
4184 }
4185
4186
4187
4188
4189
4190
4191 public void waitUntilNoRegionsInTransition(final long timeout) throws IOException {
4192 waitFor(timeout, predicateNoRegionsInTransition());
4193 }
4194
4195
4196
4197
4198
4199 public void waitUntilNoRegionsInTransition() throws IOException {
4200 waitUntilNoRegionsInTransition(15 * 60000);
4201 }
4202
4203
4204
4205
4206
4207
4208 public void waitLabelAvailable(long timeoutMillis, final String... labels) {
4209 final VisibilityLabelsCache labelsCache = VisibilityLabelsCache.get();
4210 waitFor(timeoutMillis, new Waiter.ExplainingPredicate<RuntimeException>() {
4211
4212 @Override
4213 public boolean evaluate() {
4214 for (String label : labels) {
4215 if (labelsCache.getLabelOrdinal(label) == 0) {
4216 return false;
4217 }
4218 }
4219 return true;
4220 }
4221
4222 @Override
4223 public String explainFailure() {
4224 for (String label : labels) {
4225 if (labelsCache.getLabelOrdinal(label) == 0) {
4226 return label + " is not available yet";
4227 }
4228 }
4229 return "";
4230 }
4231 });
4232 }
4233
4234
4235
4236
4237
4238
4239 public static List<HColumnDescriptor> generateColumnDescriptors() {
4240 return generateColumnDescriptors("");
4241 }
4242
4243
4244
4245
4246
4247
4248
4249 public static List<HColumnDescriptor> generateColumnDescriptors(final String prefix) {
4250 List<HColumnDescriptor> htds = new ArrayList<HColumnDescriptor>();
4251 long familyId = 0;
4252 for (Compression.Algorithm compressionType: getSupportedCompressionAlgorithms()) {
4253 for (DataBlockEncoding encodingType: DataBlockEncoding.values()) {
4254 for (BloomType bloomType: BloomType.values()) {
4255 String name = String.format("%s-cf-!@#&-%d!@#", prefix, familyId);
4256 HColumnDescriptor htd = new HColumnDescriptor(name);
4257 htd.setCompressionType(compressionType);
4258 htd.setDataBlockEncoding(encodingType);
4259 htd.setBloomFilterType(bloomType);
4260 htds.add(htd);
4261 familyId++;
4262 }
4263 }
4264 }
4265 return htds;
4266 }
4267
4268
4269
4270
4271
4272 public static Compression.Algorithm[] getSupportedCompressionAlgorithms() {
4273 String[] allAlgos = HFile.getSupportedCompressionAlgorithms();
4274 List<Compression.Algorithm> supportedAlgos = new ArrayList<Compression.Algorithm>();
4275 for (String algoName : allAlgos) {
4276 try {
4277 Compression.Algorithm algo = Compression.getCompressionAlgorithmByName(algoName);
4278 algo.getCompressor();
4279 supportedAlgos.add(algo);
4280 } catch (Throwable t) {
4281
4282 }
4283 }
4284 return supportedAlgos.toArray(new Algorithm[supportedAlgos.size()]);
4285 }
4286
4287
4288
4289
4290
4291
4292
4293 public MiniKdc setupMiniKdc(File keytabFile) throws Exception {
4294 Properties conf = MiniKdc.createConf();
4295 conf.put(MiniKdc.DEBUG, true);
4296 MiniKdc kdc = null;
4297 File dir = null;
4298
4299
4300 boolean bindException;
4301 int numTries = 0;
4302 do {
4303 try {
4304 bindException = false;
4305 dir = new File(getDataTestDir("kdc").toUri().getPath());
4306 kdc = new MiniKdc(conf, dir);
4307 kdc.start();
4308 } catch (BindException e) {
4309 FileUtils.deleteDirectory(dir);
4310 numTries++;
4311 if (numTries == 3) {
4312 LOG.error("Failed setting up MiniKDC. Tried " + numTries + " times.");
4313 throw e;
4314 }
4315 LOG.error("BindException encountered when setting up MiniKdc. Trying again.");
4316 bindException = true;
4317 }
4318 } while (bindException);
4319 HBaseKerberosUtils.setKeytabFileForTesting(keytabFile.getAbsolutePath());
4320 return kdc;
4321 }
4322 }