1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22 import static org.junit.Assert.fail;
23
24 import java.io.File;
25 import java.io.IOException;
26 import java.io.OutputStream;
27 import java.lang.reflect.Field;
28 import java.lang.reflect.Modifier;
29 import java.net.BindException;
30 import java.net.DatagramSocket;
31 import java.net.InetAddress;
32 import java.net.ServerSocket;
33 import java.net.Socket;
34 import java.net.UnknownHostException;
35 import java.security.MessageDigest;
36 import java.util.ArrayList;
37 import java.util.Arrays;
38 import java.util.Collection;
39 import java.util.Collections;
40 import java.util.HashSet;
41 import java.util.List;
42 import java.util.Map;
43 import java.util.NavigableSet;
44 import java.util.Properties;
45 import java.util.Random;
46 import java.util.Set;
47 import java.util.TreeSet;
48 import java.util.UUID;
49 import java.util.concurrent.TimeUnit;
50
51 import org.apache.commons.io.FileUtils;
52 import org.apache.commons.lang.RandomStringUtils;
53 import org.apache.commons.logging.Log;
54 import org.apache.commons.logging.LogFactory;
55 import org.apache.commons.logging.impl.Jdk14Logger;
56 import org.apache.commons.logging.impl.Log4JLogger;
57 import org.apache.hadoop.conf.Configuration;
58 import org.apache.hadoop.fs.FileSystem;
59 import org.apache.hadoop.fs.Path;
60 import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
61 import org.apache.hadoop.hbase.Waiter.Predicate;
62 import org.apache.hadoop.hbase.classification.InterfaceAudience;
63 import org.apache.hadoop.hbase.classification.InterfaceStability;
64 import org.apache.hadoop.hbase.client.Admin;
65 import org.apache.hadoop.hbase.client.Connection;
66 import org.apache.hadoop.hbase.client.ConnectionFactory;
67 import org.apache.hadoop.hbase.client.Consistency;
68 import org.apache.hadoop.hbase.client.Delete;
69 import org.apache.hadoop.hbase.client.Durability;
70 import org.apache.hadoop.hbase.client.Get;
71 import org.apache.hadoop.hbase.client.HBaseAdmin;
72 import org.apache.hadoop.hbase.client.HConnection;
73 import org.apache.hadoop.hbase.client.HTable;
74 import org.apache.hadoop.hbase.client.Put;
75 import org.apache.hadoop.hbase.client.RegionLocator;
76 import org.apache.hadoop.hbase.client.Result;
77 import org.apache.hadoop.hbase.client.ResultScanner;
78 import org.apache.hadoop.hbase.client.Scan;
79 import org.apache.hadoop.hbase.client.Table;
80 import org.apache.hadoop.hbase.fs.HFileSystem;
81 import org.apache.hadoop.hbase.io.compress.Compression;
82 import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
83 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
84 import org.apache.hadoop.hbase.io.hfile.ChecksumUtil;
85 import org.apache.hadoop.hbase.io.hfile.HFile;
86 import org.apache.hadoop.hbase.ipc.RpcServerInterface;
87 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
88 import org.apache.hadoop.hbase.mapreduce.MapreduceTestingShim;
89 import org.apache.hadoop.hbase.master.AssignmentManager;
90 import org.apache.hadoop.hbase.master.HMaster;
91 import org.apache.hadoop.hbase.master.RegionStates;
92 import org.apache.hadoop.hbase.master.ServerManager;
93 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
94 import org.apache.hadoop.hbase.regionserver.BloomType;
95 import org.apache.hadoop.hbase.regionserver.HRegion;
96 import org.apache.hadoop.hbase.regionserver.HRegionServer;
97 import org.apache.hadoop.hbase.regionserver.HStore;
98 import org.apache.hadoop.hbase.regionserver.InternalScanner;
99 import org.apache.hadoop.hbase.regionserver.Region;
100 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
101 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
102 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
103 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
104 import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
105 import org.apache.hadoop.hbase.security.User;
106 import org.apache.hadoop.hbase.security.visibility.VisibilityLabelsCache;
107 import org.apache.hadoop.hbase.util.Bytes;
108 import org.apache.hadoop.hbase.util.FSTableDescriptors;
109 import org.apache.hadoop.hbase.util.FSUtils;
110 import org.apache.hadoop.hbase.util.JVMClusterUtil;
111 import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
112 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
113 import org.apache.hadoop.hbase.util.Pair;
114 import org.apache.hadoop.hbase.util.RegionSplitter;
115 import org.apache.hadoop.hbase.util.RetryCounter;
116 import org.apache.hadoop.hbase.util.Threads;
117 import org.apache.hadoop.hbase.wal.WAL;
118 import org.apache.hadoop.hbase.wal.WALFactory;
119 import org.apache.hadoop.hbase.zookeeper.EmptyWatcher;
120 import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
121 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
122 import org.apache.hadoop.hbase.zookeeper.ZKConfig;
123 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
124 import org.apache.hadoop.hdfs.DFSClient;
125 import org.apache.hadoop.hdfs.DistributedFileSystem;
126 import org.apache.hadoop.hdfs.MiniDFSCluster;
127 import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
128 import org.apache.hadoop.mapred.JobConf;
129 import org.apache.hadoop.mapred.MiniMRCluster;
130 import org.apache.hadoop.mapred.TaskLog;
131 import org.apache.hadoop.minikdc.MiniKdc;
132 import org.apache.zookeeper.KeeperException;
133 import org.apache.zookeeper.KeeperException.NodeExistsException;
134 import org.apache.zookeeper.WatchedEvent;
135 import org.apache.zookeeper.ZooKeeper;
136 import org.apache.zookeeper.ZooKeeper.States;
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152 @InterfaceAudience.Public
153 @InterfaceStability.Evolving
154 @SuppressWarnings("deprecation")
155 public class HBaseTestingUtility extends HBaseCommonTestingUtility {
156 private MiniZooKeeperCluster zkCluster = null;
157
158 public static final String REGIONS_PER_SERVER_KEY = "hbase.test.regions-per-server";
159
160
161
162
163 public static final int DEFAULT_REGIONS_PER_SERVER = 3;
164
165
166 public static final String PRESPLIT_TEST_TABLE_KEY = "hbase.test.pre-split-table";
167 public static final boolean PRESPLIT_TEST_TABLE = true;
168
169 public static final String USE_LOCAL_FILESYSTEM = "hbase.test.local.fileSystem";
170
171
172
173
174 private boolean passedZkCluster = false;
175 private MiniDFSCluster dfsCluster = null;
176
177 private volatile HBaseCluster hbaseCluster = null;
178 private MiniMRCluster mrCluster = null;
179
180
181 private volatile boolean miniClusterRunning;
182
183 private String hadoopLogDir;
184
185
186 private File clusterTestDir = null;
187
188
189
190 private Path dataTestDirOnTestFS = null;
191
192
193
194
195 private volatile Connection connection;
196
197
198
199
200
201
202
203
204 @Deprecated
205 private static final String TEST_DIRECTORY_KEY = "test.build.data";
206
207
208 private static String FS_URI;
209
210
211 private static final Set<Integer> takenRandomPorts = new HashSet<Integer>();
212
213
214 public static final List<Object[]> COMPRESSION_ALGORITHMS_PARAMETERIZED =
215 Arrays.asList(new Object[][] {
216 { Compression.Algorithm.NONE },
217 { Compression.Algorithm.GZ }
218 });
219
220
221 public static final List<Object[]> BOOLEAN_PARAMETERIZED =
222 Arrays.asList(new Object[][] {
223 { new Boolean(false) },
224 { new Boolean(true) }
225 });
226
227
228 public static final List<Object[]> MEMSTORETS_TAGS_PARAMETRIZED = memStoreTSAndTagsCombination() ;
229
230 public static final Compression.Algorithm[] COMPRESSION_ALGORITHMS ={
231 Compression.Algorithm.NONE, Compression.Algorithm.GZ
232 };
233
234
235
236
237
238
239
240 public static boolean available(int port) {
241 ServerSocket ss = null;
242 DatagramSocket ds = null;
243 try {
244 ss = new ServerSocket(port);
245 ss.setReuseAddress(true);
246 ds = new DatagramSocket(port);
247 ds.setReuseAddress(true);
248 return true;
249 } catch (IOException e) {
250
251 } finally {
252 if (ds != null) {
253 ds.close();
254 }
255
256 if (ss != null) {
257 try {
258 ss.close();
259 } catch (IOException e) {
260
261 }
262 }
263 }
264
265 return false;
266 }
267
268
269
270
271
272 private static List<Object[]> bloomAndCompressionCombinations() {
273 List<Object[]> configurations = new ArrayList<Object[]>();
274 for (Compression.Algorithm comprAlgo :
275 HBaseTestingUtility.COMPRESSION_ALGORITHMS) {
276 for (BloomType bloomType : BloomType.values()) {
277 configurations.add(new Object[] { comprAlgo, bloomType });
278 }
279 }
280 return Collections.unmodifiableList(configurations);
281 }
282
283
284
285
286 private static List<Object[]> memStoreTSAndTagsCombination() {
287 List<Object[]> configurations = new ArrayList<Object[]>();
288 configurations.add(new Object[] { false, false });
289 configurations.add(new Object[] { false, true });
290 configurations.add(new Object[] { true, false });
291 configurations.add(new Object[] { true, true });
292 return Collections.unmodifiableList(configurations);
293 }
294
295 public static final Collection<Object[]> BLOOM_AND_COMPRESSION_COMBINATIONS =
296 bloomAndCompressionCombinations();
297
298 public HBaseTestingUtility() {
299 this(HBaseConfiguration.create());
300 }
301
302 public HBaseTestingUtility(Configuration conf) {
303 super(conf);
304
305
306 ChecksumUtil.generateExceptionForChecksumFailureForTest(true);
307 }
308
309
310
311
312
313
314
315 public static HBaseTestingUtility createLocalHTU() {
316 Configuration c = HBaseConfiguration.create();
317 return createLocalHTU(c);
318 }
319
320
321
322
323
324
325
326
327 public static HBaseTestingUtility createLocalHTU(Configuration c) {
328 HBaseTestingUtility htu = new HBaseTestingUtility(c);
329 String dataTestDir = htu.getDataTestDir().toString();
330 htu.getConfiguration().set(HConstants.HBASE_DIR, dataTestDir);
331 LOG.debug("Setting " + HConstants.HBASE_DIR + " to " + dataTestDir);
332 return htu;
333 }
334
335
336
337
338 public static void closeRegion(final Region r) throws IOException {
339 if (r != null) {
340 ((HRegion)r).close();
341 }
342 }
343
344
345
346
347
348
349
350
351
352
353
354
355 @Override
356 public Configuration getConfiguration() {
357 return super.getConfiguration();
358 }
359
360 public void setHBaseCluster(HBaseCluster hbaseCluster) {
361 this.hbaseCluster = hbaseCluster;
362 }
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380 @Override
381 protected Path setupDataTestDir() {
382 Path testPath = super.setupDataTestDir();
383 if (null == testPath) {
384 return null;
385 }
386
387 createSubDirAndSystemProperty(
388 "hadoop.log.dir",
389 testPath, "hadoop-log-dir");
390
391
392
393 createSubDirAndSystemProperty(
394 "hadoop.tmp.dir",
395 testPath, "hadoop-tmp-dir");
396
397
398 createSubDir(
399 "mapreduce.cluster.local.dir",
400 testPath, "mapred-local-dir");
401
402 return testPath;
403 }
404
405 public void setJobWithoutMRCluster() throws IOException {
406 conf.set("hbase.fs.tmp.dir", getDataTestDirOnTestFS("hbase-staging").toString());
407 conf.setBoolean(HBaseTestingUtility.USE_LOCAL_FILESYSTEM, true);
408 }
409
410 private void createSubDirAndSystemProperty(
411 String propertyName, Path parent, String subDirName){
412
413 String sysValue = System.getProperty(propertyName);
414
415 if (sysValue != null) {
416
417
418 LOG.info("System.getProperty(\""+propertyName+"\") already set to: "+
419 sysValue + " so I do NOT create it in " + parent);
420 String confValue = conf.get(propertyName);
421 if (confValue != null && !confValue.endsWith(sysValue)){
422 LOG.warn(
423 propertyName + " property value differs in configuration and system: "+
424 "Configuration="+confValue+" while System="+sysValue+
425 " Erasing configuration value by system value."
426 );
427 }
428 conf.set(propertyName, sysValue);
429 } else {
430
431 createSubDir(propertyName, parent, subDirName);
432 System.setProperty(propertyName, conf.get(propertyName));
433 }
434 }
435
436
437
438
439
440
441
442 private Path getBaseTestDirOnTestFS() throws IOException {
443 FileSystem fs = getTestFileSystem();
444 return new Path(fs.getWorkingDirectory(), "test-data");
445 }
446
447
448
449
450 public HTableDescriptor getMetaTableDescriptor() {
451 try {
452 return new FSTableDescriptors(conf).get(TableName.META_TABLE_NAME);
453 } catch (IOException e) {
454 throw new RuntimeException("Unable to create META table descriptor", e);
455 }
456 }
457
458
459
460
461
462
463 Path getClusterTestDir() {
464 if (clusterTestDir == null){
465 setupClusterTestDir();
466 }
467 return new Path(clusterTestDir.getAbsolutePath());
468 }
469
470
471
472
473 private void setupClusterTestDir() {
474 if (clusterTestDir != null) {
475 return;
476 }
477
478
479
480 Path testDir = getDataTestDir("dfscluster_" + UUID.randomUUID().toString());
481 clusterTestDir = new File(testDir.toString()).getAbsoluteFile();
482
483 boolean b = deleteOnExit();
484 if (b) clusterTestDir.deleteOnExit();
485 conf.set(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
486 LOG.info("Created new mini-cluster data directory: " + clusterTestDir + ", deleteOnExit=" + b);
487 }
488
489
490
491
492
493
494
495 public Path getDataTestDirOnTestFS() throws IOException {
496 if (dataTestDirOnTestFS == null) {
497 setupDataTestDirOnTestFS();
498 }
499
500 return dataTestDirOnTestFS;
501 }
502
503
504
505
506
507
508
509
510 public Path getDataTestDirOnTestFS(final String subdirName) throws IOException {
511 return new Path(getDataTestDirOnTestFS(), subdirName);
512 }
513
514
515
516
517
518 private void setupDataTestDirOnTestFS() throws IOException {
519 if (dataTestDirOnTestFS != null) {
520 LOG.warn("Data test on test fs dir already setup in "
521 + dataTestDirOnTestFS.toString());
522 return;
523 }
524 dataTestDirOnTestFS = getNewDataTestDirOnTestFS();
525 }
526
527
528
529
530 private Path getNewDataTestDirOnTestFS() throws IOException {
531
532
533
534
535 FileSystem fs = getTestFileSystem();
536 Path newDataTestDir = null;
537 String randomStr = UUID.randomUUID().toString();
538 if (fs.getUri().getScheme().equals(FileSystem.getLocal(conf).getUri().getScheme())) {
539 newDataTestDir = new Path(getDataTestDir(), randomStr);
540 File dataTestDir = new File(newDataTestDir.toString());
541 if (deleteOnExit()) dataTestDir.deleteOnExit();
542 } else {
543 Path base = getBaseTestDirOnTestFS();
544 newDataTestDir = new Path(base, randomStr);
545 if (deleteOnExit()) fs.deleteOnExit(newDataTestDir);
546 }
547 return newDataTestDir;
548 }
549
550
551
552
553
554
555 public boolean cleanupDataTestDirOnTestFS() throws IOException {
556 boolean ret = getTestFileSystem().delete(dataTestDirOnTestFS, true);
557 if (ret)
558 dataTestDirOnTestFS = null;
559 return ret;
560 }
561
562
563
564
565
566
567 public boolean cleanupDataTestDirOnTestFS(String subdirName) throws IOException {
568 Path cpath = getDataTestDirOnTestFS(subdirName);
569 return getTestFileSystem().delete(cpath, true);
570 }
571
572
573
574
575
576
577
578
579 public MiniDFSCluster startMiniDFSCluster(int servers) throws Exception {
580 return startMiniDFSCluster(servers, null);
581 }
582
583
584
585
586
587
588
589
590
591
592
593
594 public MiniDFSCluster startMiniDFSCluster(final String hosts[])
595 throws Exception {
596 if ( hosts != null && hosts.length != 0) {
597 return startMiniDFSCluster(hosts.length, hosts);
598 } else {
599 return startMiniDFSCluster(1, null);
600 }
601 }
602
603
604
605
606
607
608
609
610
611
612 public MiniDFSCluster startMiniDFSCluster(int servers, final String hosts[])
613 throws Exception {
614 createDirsAndSetProperties();
615 EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
616
617
618 org.apache.log4j.Logger.getLogger(org.apache.hadoop.metrics2.util.MBeans.class).
619 setLevel(org.apache.log4j.Level.ERROR);
620 org.apache.log4j.Logger.getLogger(org.apache.hadoop.metrics2.impl.MetricsSystemImpl.class).
621 setLevel(org.apache.log4j.Level.ERROR);
622
623
624 this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true,
625 true, null, null, hosts, null);
626
627
628 setFs();
629
630
631 this.dfsCluster.waitClusterUp();
632
633
634 dataTestDirOnTestFS = null;
635
636 return this.dfsCluster;
637 }
638
639 private void setFs() throws IOException {
640 if(this.dfsCluster == null){
641 LOG.info("Skipping setting fs because dfsCluster is null");
642 return;
643 }
644 FileSystem fs = this.dfsCluster.getFileSystem();
645 FSUtils.setFsDefault(this.conf, new Path(fs.getUri()));
646 if (this.conf.getBoolean(USE_LOCAL_FILESYSTEM, false)) {
647 FSUtils.setFsDefault(this.conf, new Path("file:///"));
648 }
649 }
650
651 public MiniDFSCluster startMiniDFSCluster(int servers, final String racks[], String hosts[])
652 throws Exception {
653 createDirsAndSetProperties();
654 this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true,
655 true, null, racks, hosts, null);
656
657
658 FileSystem fs = this.dfsCluster.getFileSystem();
659 FSUtils.setFsDefault(this.conf, new Path(fs.getUri()));
660
661
662 this.dfsCluster.waitClusterUp();
663
664
665 dataTestDirOnTestFS = null;
666
667 return this.dfsCluster;
668 }
669
670 public MiniDFSCluster startMiniDFSClusterForTestWAL(int namenodePort) throws IOException {
671 createDirsAndSetProperties();
672 dfsCluster = new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null,
673 null, null, null);
674 return dfsCluster;
675 }
676
677
678 private void createDirsAndSetProperties() throws IOException {
679 setupClusterTestDir();
680 System.setProperty(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
681 createDirAndSetProperty("cache_data", "test.cache.data");
682 createDirAndSetProperty("hadoop_tmp", "hadoop.tmp.dir");
683 hadoopLogDir = createDirAndSetProperty("hadoop_logs", "hadoop.log.dir");
684 createDirAndSetProperty("mapred_local", "mapreduce.cluster.local.dir");
685 createDirAndSetProperty("mapred_temp", "mapreduce.cluster.temp.dir");
686 enableShortCircuit();
687
688 Path root = getDataTestDirOnTestFS("hadoop");
689 conf.set(MapreduceTestingShim.getMROutputDirProp(),
690 new Path(root, "mapred-output-dir").toString());
691 conf.set("mapreduce.jobtracker.system.dir", new Path(root, "mapred-system-dir").toString());
692 conf.set("mapreduce.jobtracker.staging.root.dir",
693 new Path(root, "mapreduce-jobtracker-staging-root-dir").toString());
694 conf.set("mapreduce.job.working.dir", new Path(root, "mapred-working-dir").toString());
695 }
696
697
698
699
700
701
702
703 public boolean isReadShortCircuitOn(){
704 final String propName = "hbase.tests.use.shortcircuit.reads";
705 String readOnProp = System.getProperty(propName);
706 if (readOnProp != null){
707 return Boolean.parseBoolean(readOnProp);
708 } else {
709 return conf.getBoolean(propName, false);
710 }
711 }
712
713
714
715
716 private void enableShortCircuit() {
717 if (isReadShortCircuitOn()) {
718 String curUser = System.getProperty("user.name");
719 LOG.info("read short circuit is ON for user " + curUser);
720
721 conf.set("dfs.block.local-path-access.user", curUser);
722
723 conf.setBoolean("dfs.client.read.shortcircuit", true);
724
725 conf.setBoolean("dfs.client.read.shortcircuit.skip.checksum", true);
726 } else {
727 LOG.info("read short circuit is OFF");
728 }
729 }
730
731 private String createDirAndSetProperty(final String relPath, String property) {
732 String path = getDataTestDir(relPath).toString();
733 System.setProperty(property, path);
734 conf.set(property, path);
735 new File(path).mkdirs();
736 LOG.info("Setting " + property + " to " + path + " in system properties and HBase conf");
737 return path;
738 }
739
740
741
742
743
744
745 public void shutdownMiniDFSCluster() throws IOException {
746 if (this.dfsCluster != null) {
747
748 this.dfsCluster.shutdown();
749 dfsCluster = null;
750 dataTestDirOnTestFS = null;
751 FSUtils.setFsDefault(this.conf, new Path("file:///"));
752 }
753 }
754
755
756
757
758
759
760
761
762 public MiniZooKeeperCluster startMiniZKCluster() throws Exception {
763 return startMiniZKCluster(1);
764 }
765
766
767
768
769
770
771
772
773
774 public MiniZooKeeperCluster startMiniZKCluster(
775 final int zooKeeperServerNum,
776 final int ... clientPortList)
777 throws Exception {
778 setupClusterTestDir();
779 return startMiniZKCluster(clusterTestDir, zooKeeperServerNum, clientPortList);
780 }
781
782 private MiniZooKeeperCluster startMiniZKCluster(final File dir)
783 throws Exception {
784 return startMiniZKCluster(dir, 1, null);
785 }
786
787
788
789
790
791 private MiniZooKeeperCluster startMiniZKCluster(final File dir,
792 final int zooKeeperServerNum,
793 final int [] clientPortList)
794 throws Exception {
795 if (this.zkCluster != null) {
796 throw new IOException("Cluster already running at " + dir);
797 }
798 this.passedZkCluster = false;
799 this.zkCluster = new MiniZooKeeperCluster(this.getConfiguration());
800 final int defPort = this.conf.getInt("test.hbase.zookeeper.property.clientPort", 0);
801 if (defPort > 0){
802
803 this.zkCluster.setDefaultClientPort(defPort);
804 }
805
806 if (clientPortList != null) {
807
808 int clientPortListSize = (clientPortList.length <= zooKeeperServerNum) ?
809 clientPortList.length : zooKeeperServerNum;
810 for (int i=0; i < clientPortListSize; i++) {
811 this.zkCluster.addClientPort(clientPortList[i]);
812 }
813 }
814 int clientPort = this.zkCluster.startup(dir,zooKeeperServerNum);
815 this.conf.set(HConstants.ZOOKEEPER_CLIENT_PORT,
816 Integer.toString(clientPort));
817 return this.zkCluster;
818 }
819
820
821
822
823
824
825
826 public void shutdownMiniZKCluster() throws IOException {
827 if (this.zkCluster != null) {
828 this.zkCluster.shutdown();
829 this.zkCluster = null;
830 }
831 }
832
833
834
835
836
837
838
839 public MiniHBaseCluster startMiniCluster() throws Exception {
840 return startMiniCluster(1, 1);
841 }
842
843
844
845
846
847
848
849 public MiniHBaseCluster startMiniCluster(boolean withWALDir) throws Exception {
850 return startMiniCluster(1, 1, 1, null, null, null, false, withWALDir);
851 }
852
853
854
855
856
857
858
859
860
861 public MiniHBaseCluster startMiniCluster(final int numSlaves, boolean create)
862 throws Exception {
863 return startMiniCluster(1, numSlaves, create);
864 }
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879 public MiniHBaseCluster startMiniCluster(final int numSlaves)
880 throws Exception {
881 return startMiniCluster(1, numSlaves, false);
882 }
883
884 public MiniHBaseCluster startMiniCluster(final int numSlaves, boolean create, boolean withWALDir)
885 throws Exception {
886 return startMiniCluster(1, numSlaves, numSlaves, null, null, null, create, withWALDir);
887 }
888
889
890
891
892
893
894
895
896 public MiniHBaseCluster startMiniCluster(final int numMasters,
897 final int numSlaves, boolean create)
898 throws Exception {
899 return startMiniCluster(numMasters, numSlaves, null, create);
900 }
901
902
903
904
905
906
907
908 public MiniHBaseCluster startMiniCluster(final int numMasters,
909 final int numSlaves)
910 throws Exception {
911 return startMiniCluster(numMasters, numSlaves, null, false);
912 }
913
914 public MiniHBaseCluster startMiniCluster(final int numMasters,
915 final int numSlaves, final String[] dataNodeHosts, boolean create)
916 throws Exception {
917 return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts,
918 null, null, create, false);
919 }
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945 public MiniHBaseCluster startMiniCluster(final int numMasters,
946 final int numSlaves, final String[] dataNodeHosts) throws Exception {
947 return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts,
948 null, null);
949 }
950
951
952
953
954
955 public MiniHBaseCluster startMiniCluster(final int numMasters,
956 final int numSlaves, final int numDataNodes) throws Exception {
957 return startMiniCluster(numMasters, numSlaves, numDataNodes, null, null, null);
958 }
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987 public MiniHBaseCluster startMiniCluster(final int numMasters,
988 final int numSlaves, final String[] dataNodeHosts, Class<? extends HMaster> masterClass,
989 Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
990 throws Exception {
991 return startMiniCluster(
992 numMasters, numSlaves, numSlaves, dataNodeHosts, masterClass, regionserverClass);
993 }
994
995 public MiniHBaseCluster startMiniCluster(final int numMasters,
996 final int numSlaves, int numDataNodes, final String[] dataNodeHosts,
997 Class<? extends HMaster> masterClass,
998 Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
999 throws Exception {
1000 return startMiniCluster(numMasters, numSlaves, numDataNodes, dataNodeHosts,
1001 masterClass, regionserverClass, false, false);
1002 }
1003
1004
1005
1006
1007
1008
1009
1010
1011 public MiniHBaseCluster startMiniCluster(final int numMasters,
1012 final int numSlaves, int numDataNodes, final String[] dataNodeHosts,
1013 Class<? extends HMaster> masterClass,
1014 Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass,
1015 boolean create, boolean withWALDir)
1016 throws Exception {
1017 if (dataNodeHosts != null && dataNodeHosts.length != 0) {
1018 numDataNodes = dataNodeHosts.length;
1019 }
1020
1021 LOG.info("Starting up minicluster with " + numMasters + " master(s) and " +
1022 numSlaves + " regionserver(s) and " + numDataNodes + " datanode(s)");
1023
1024
1025 if (miniClusterRunning) {
1026 throw new IllegalStateException("A mini-cluster is already running");
1027 }
1028 miniClusterRunning = true;
1029
1030 setupClusterTestDir();
1031 System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestDir.getPath());
1032
1033
1034
1035 if(this.dfsCluster == null) {
1036 dfsCluster = startMiniDFSCluster(numDataNodes, dataNodeHosts);
1037 }
1038
1039
1040 if (this.zkCluster == null) {
1041 startMiniZKCluster(clusterTestDir);
1042 }
1043
1044
1045 return startMiniHBaseCluster(numMasters, numSlaves, masterClass,
1046 regionserverClass, create, withWALDir);
1047 }
1048
1049 public MiniHBaseCluster startMiniHBaseCluster(final int numMasters, final int numSlaves)
1050 throws IOException, InterruptedException{
1051 return startMiniHBaseCluster(numMasters, numSlaves, null, null, false, false);
1052 }
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067 public MiniHBaseCluster startMiniHBaseCluster(final int numMasters,
1068 final int numSlaves, Class<? extends HMaster> masterClass,
1069 Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass,
1070 boolean create, boolean withWALDir)
1071 throws IOException, InterruptedException {
1072
1073 createRootDir(create);
1074
1075 if (withWALDir) {
1076 createWALRootDir();
1077 }
1078
1079
1080
1081 if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1) == -1) {
1082 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, numSlaves);
1083 }
1084 if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1) == -1) {
1085 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, numSlaves);
1086 }
1087
1088 Configuration c = new Configuration(this.conf);
1089 this.hbaseCluster =
1090 new MiniHBaseCluster(c, numMasters, numSlaves, masterClass, regionserverClass);
1091
1092 Table t = new HTable(c, TableName.META_TABLE_NAME);
1093 ResultScanner s = t.getScanner(new Scan());
1094 while (s.next() != null) {
1095 continue;
1096 }
1097 s.close();
1098 t.close();
1099
1100 getHBaseAdmin();
1101 LOG.info("Minicluster is up");
1102
1103
1104
1105 setHBaseFsTmpDir();
1106
1107 return (MiniHBaseCluster)this.hbaseCluster;
1108 }
1109
1110
1111
1112
1113
1114
1115
1116 public void restartHBaseCluster(int servers) throws IOException, InterruptedException {
1117 this.hbaseCluster = new MiniHBaseCluster(this.conf, servers);
1118
1119 Table t = new HTable(new Configuration(this.conf), TableName.META_TABLE_NAME);
1120 ResultScanner s = t.getScanner(new Scan());
1121 while (s.next() != null) {
1122
1123 }
1124 LOG.info("HBase has been restarted");
1125 s.close();
1126 t.close();
1127 }
1128
1129
1130
1131
1132
1133
1134 public MiniHBaseCluster getMiniHBaseCluster() {
1135 if (this.hbaseCluster == null || this.hbaseCluster instanceof MiniHBaseCluster) {
1136 return (MiniHBaseCluster)this.hbaseCluster;
1137 }
1138 throw new RuntimeException(hbaseCluster + " not an instance of " +
1139 MiniHBaseCluster.class.getName());
1140 }
1141
1142
1143
1144
1145
1146
1147 public void shutdownMiniCluster() throws Exception {
1148 LOG.info("Shutting down minicluster");
1149 if (this.connection != null && !this.connection.isClosed()) {
1150 this.connection.close();
1151 this.connection = null;
1152 }
1153 shutdownMiniHBaseCluster();
1154 if (!this.passedZkCluster){
1155 shutdownMiniZKCluster();
1156 }
1157 shutdownMiniDFSCluster();
1158
1159 cleanupTestDir();
1160 miniClusterRunning = false;
1161 LOG.info("Minicluster is down");
1162 }
1163
1164
1165
1166
1167
1168 @Override
1169 public boolean cleanupTestDir() throws IOException {
1170 boolean ret = super.cleanupTestDir();
1171 if (deleteDir(this.clusterTestDir)) {
1172 this.clusterTestDir = null;
1173 return ret & true;
1174 }
1175 return false;
1176 }
1177
1178
1179
1180
1181
1182 public void shutdownMiniHBaseCluster() throws IOException {
1183 if (hbaseAdmin != null) {
1184 hbaseAdmin.close0();
1185 hbaseAdmin = null;
1186 }
1187
1188
1189 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1);
1190 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1);
1191 if (this.hbaseCluster != null) {
1192 this.hbaseCluster.shutdown();
1193
1194 this.hbaseCluster.waitUntilShutDown();
1195 this.hbaseCluster = null;
1196 }
1197
1198 if (zooKeeperWatcher != null) {
1199 zooKeeperWatcher.close();
1200 zooKeeperWatcher = null;
1201 }
1202 }
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212 public Path getDefaultRootDirPath(boolean create) throws IOException {
1213 if (!create) {
1214 return getDataTestDirOnTestFS();
1215 } else {
1216 return getNewDataTestDirOnTestFS();
1217 }
1218 }
1219
1220
1221
1222
1223
1224
1225
1226
1227 public Path getDefaultRootDirPath() throws IOException {
1228 return getDefaultRootDirPath(false);
1229 }
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243 public Path createRootDir(boolean create) throws IOException {
1244 FileSystem fs = FileSystem.get(this.conf);
1245 Path hbaseRootdir = getDefaultRootDirPath(create);
1246 FSUtils.setRootDir(this.conf, hbaseRootdir);
1247 fs.mkdirs(hbaseRootdir);
1248 FSUtils.setVersion(fs, hbaseRootdir);
1249 return hbaseRootdir;
1250 }
1251
1252
1253
1254
1255
1256
1257
1258 public Path createRootDir() throws IOException {
1259 return createRootDir(false);
1260 }
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271 public Path createWALRootDir() throws IOException {
1272 FileSystem fs = FileSystem.get(this.conf);
1273 Path walDir = getNewDataTestDirOnTestFS();
1274 FSUtils.setWALRootDir(this.conf, walDir);
1275 fs.mkdirs(walDir);
1276 return walDir;
1277 }
1278
1279 private void setHBaseFsTmpDir() throws IOException {
1280 String hbaseFsTmpDirInString = this.conf.get("hbase.fs.tmp.dir");
1281 if (hbaseFsTmpDirInString == null) {
1282 this.conf.set("hbase.fs.tmp.dir", getDataTestDirOnTestFS("hbase-staging").toString());
1283 LOG.info("Setting hbase.fs.tmp.dir to " + this.conf.get("hbase.fs.tmp.dir"));
1284 } else {
1285 LOG.info("The hbase.fs.tmp.dir is set to " + hbaseFsTmpDirInString);
1286 }
1287 }
1288
1289
1290
1291
1292
1293 public void flush() throws IOException {
1294 getMiniHBaseCluster().flushcache();
1295 }
1296
1297
1298
1299
1300
1301 public void flush(TableName tableName) throws IOException {
1302 getMiniHBaseCluster().flushcache(tableName);
1303 }
1304
1305
1306
1307
1308
1309 public void compact(boolean major) throws IOException {
1310 getMiniHBaseCluster().compact(major);
1311 }
1312
1313
1314
1315
1316
1317 public void compact(TableName tableName, boolean major) throws IOException {
1318 getMiniHBaseCluster().compact(tableName, major);
1319 }
1320
1321
1322
1323
1324
1325
1326
1327
1328 public Table createTable(TableName tableName, String family)
1329 throws IOException{
1330 return createTable(tableName, new String[]{family});
1331 }
1332
1333
1334
1335
1336
1337
1338
1339
1340 public HTable createTable(byte[] tableName, byte[] family)
1341 throws IOException{
1342 return createTable(TableName.valueOf(tableName), new byte[][]{family});
1343 }
1344
1345
1346
1347
1348
1349
1350
1351
1352 public Table createTable(TableName tableName, String[] families)
1353 throws IOException {
1354 List<byte[]> fams = new ArrayList<byte[]>(families.length);
1355 for (String family : families) {
1356 fams.add(Bytes.toBytes(family));
1357 }
1358 return createTable(tableName, fams.toArray(new byte[0][]));
1359 }
1360
1361
1362
1363
1364
1365
1366
1367
1368 public HTable createTable(TableName tableName, byte[] family)
1369 throws IOException{
1370 return createTable(tableName, new byte[][]{family});
1371 }
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381 public HTable createMultiRegionTable(TableName tableName, byte[] family, int numRegions)
1382 throws IOException {
1383 if (numRegions < 3) throw new IOException("Must create at least 3 regions");
1384 byte[] startKey = Bytes.toBytes("aaaaa");
1385 byte[] endKey = Bytes.toBytes("zzzzz");
1386 byte[][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
1387
1388 return createTable(tableName, new byte[][] { family }, splitKeys);
1389 }
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399 public HTable createTable(byte[] tableName, byte[][] families)
1400 throws IOException {
1401 return createTable(tableName, families,
1402 new Configuration(getConfiguration()));
1403 }
1404
1405
1406
1407
1408
1409
1410
1411
1412 public HTable createTable(TableName tableName, byte[][] families)
1413 throws IOException {
1414 return createTable(tableName, families, (byte[][]) null);
1415 }
1416
1417
1418
1419
1420
1421
1422
1423
1424 public HTable createMultiRegionTable(TableName tableName, byte[][] families) throws IOException {
1425 return createTable(tableName, families, KEYS_FOR_HBA_CREATE_TABLE);
1426 }
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436 public HTable createTable(TableName tableName, byte[][] families, byte[][] splitKeys)
1437 throws IOException {
1438 return createTable(tableName, families, splitKeys, new Configuration(getConfiguration()));
1439 }
1440
1441 public HTable createTable(byte[] tableName, byte[][] families,
1442 int numVersions, byte[] startKey, byte[] endKey, int numRegions) throws IOException {
1443 return createTable(TableName.valueOf(tableName), families, numVersions,
1444 startKey, endKey, numRegions);
1445 }
1446
1447 public HTable createTable(String tableName, byte[][] families,
1448 int numVersions, byte[] startKey, byte[] endKey, int numRegions) throws IOException {
1449 return createTable(TableName.valueOf(tableName), families, numVersions,
1450 startKey, endKey, numRegions);
1451 }
1452
1453 public HTable createTable(TableName tableName, byte[][] families,
1454 int numVersions, byte[] startKey, byte[] endKey, int numRegions)
1455 throws IOException{
1456 HTableDescriptor desc = new HTableDescriptor(tableName);
1457 for (byte[] family : families) {
1458 HColumnDescriptor hcd = new HColumnDescriptor(family)
1459 .setMaxVersions(numVersions);
1460 desc.addFamily(hcd);
1461 }
1462 getHBaseAdmin().createTable(desc, startKey, endKey, numRegions);
1463
1464 waitUntilAllRegionsAssigned(tableName);
1465 return new HTable(getConfiguration(), tableName);
1466 }
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476 public HTable createTable(HTableDescriptor htd, byte[][] families, Configuration c)
1477 throws IOException {
1478 return createTable(htd, families, (byte[][]) null, c);
1479 }
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490 public HTable createTable(HTableDescriptor htd, byte[][] families, byte[][] splitKeys,
1491 Configuration c) throws IOException {
1492
1493
1494
1495 return createTable(htd, families, splitKeys, BloomType.NONE, HConstants.DEFAULT_BLOCKSIZE, c);
1496 }
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510 public HTable createTable(HTableDescriptor htd, byte[][] families, byte[][] splitKeys,
1511 BloomType type, int blockSize, Configuration c) throws IOException {
1512 for (byte[] family : families) {
1513 HColumnDescriptor hcd = new HColumnDescriptor(family);
1514
1515
1516
1517 hcd.setBloomFilterType(type);
1518 hcd.setBlocksize(blockSize);
1519 htd.addFamily(hcd);
1520 }
1521 getHBaseAdmin().createTable(htd, splitKeys);
1522
1523
1524 waitUntilAllRegionsAssigned(htd.getTableName());
1525 return (HTable) getConnection().getTable(htd.getTableName());
1526 }
1527
1528
1529
1530
1531
1532
1533
1534
1535 public HTable createTable(HTableDescriptor htd, byte[][] splitRows)
1536 throws IOException {
1537 getHBaseAdmin().createTable(htd, splitRows);
1538
1539 waitUntilAllRegionsAssigned(htd.getTableName());
1540 return new HTable(getConfiguration(), htd.getTableName());
1541 }
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551 public HTable createTable(TableName tableName, byte[][] families,
1552 final Configuration c)
1553 throws IOException {
1554 return createTable(tableName, families, (byte[][]) null, c);
1555 }
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566 public HTable createTable(TableName tableName, byte[][] families, byte[][] splitKeys,
1567 final Configuration c) throws IOException {
1568 return createTable(new HTableDescriptor(tableName), families, splitKeys, c);
1569 }
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579 public HTable createTable(byte[] tableName, byte[][] families,
1580 final Configuration c)
1581 throws IOException {
1582 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
1583 for(byte[] family : families) {
1584 HColumnDescriptor hcd = new HColumnDescriptor(family);
1585
1586
1587
1588 hcd.setBloomFilterType(BloomType.NONE);
1589 desc.addFamily(hcd);
1590 }
1591 getHBaseAdmin().createTable(desc);
1592 return new HTable(c, desc.getTableName());
1593 }
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604 public HTable createTable(TableName tableName, byte[][] families,
1605 final Configuration c, int numVersions)
1606 throws IOException {
1607 HTableDescriptor desc = new HTableDescriptor(tableName);
1608 for(byte[] family : families) {
1609 HColumnDescriptor hcd = new HColumnDescriptor(family)
1610 .setMaxVersions(numVersions);
1611 desc.addFamily(hcd);
1612 }
1613 getHBaseAdmin().createTable(desc);
1614
1615 waitUntilAllRegionsAssigned(tableName);
1616 return new HTable(c, tableName);
1617 }
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628 public HTable createTable(byte[] tableName, byte[][] families,
1629 final Configuration c, int numVersions)
1630 throws IOException {
1631 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
1632 for(byte[] family : families) {
1633 HColumnDescriptor hcd = new HColumnDescriptor(family)
1634 .setMaxVersions(numVersions);
1635 desc.addFamily(hcd);
1636 }
1637 getHBaseAdmin().createTable(desc);
1638 return new HTable(c, desc.getTableName());
1639 }
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649 public HTable createTable(byte[] tableName, byte[] family, int numVersions)
1650 throws IOException {
1651 return createTable(tableName, new byte[][]{family}, numVersions);
1652 }
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662 public HTable createTable(TableName tableName, byte[] family, int numVersions)
1663 throws IOException {
1664 return createTable(tableName, new byte[][]{family}, numVersions);
1665 }
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675 public HTable createTable(byte[] tableName, byte[][] families,
1676 int numVersions)
1677 throws IOException {
1678 return createTable(TableName.valueOf(tableName), families, numVersions);
1679 }
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689 public HTable createTable(TableName tableName, byte[][] families,
1690 int numVersions)
1691 throws IOException {
1692 return createTable(tableName, families, numVersions, (byte[][]) null);
1693 }
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704 public HTable createTable(TableName tableName, byte[][] families, int numVersions,
1705 byte[][] splitKeys) throws IOException {
1706 HTableDescriptor desc = new HTableDescriptor(tableName);
1707 for (byte[] family : families) {
1708 HColumnDescriptor hcd = new HColumnDescriptor(family).setMaxVersions(numVersions);
1709 desc.addFamily(hcd);
1710 }
1711 getHBaseAdmin().createTable(desc, splitKeys);
1712
1713 waitUntilAllRegionsAssigned(tableName);
1714 return new HTable(new Configuration(getConfiguration()), tableName);
1715 }
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725 public HTable createMultiRegionTable(TableName tableName, byte[][] families, int numVersions)
1726 throws IOException {
1727 return createTable(tableName, families, numVersions, KEYS_FOR_HBA_CREATE_TABLE);
1728 }
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739 public HTable createTable(byte[] tableName, byte[][] families,
1740 int numVersions, int blockSize) throws IOException {
1741 return createTable(TableName.valueOf(tableName),
1742 families, numVersions, blockSize);
1743 }
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754 public HTable createTable(TableName tableName, byte[][] families,
1755 int numVersions, int blockSize) throws IOException {
1756 HTableDescriptor desc = new HTableDescriptor(tableName);
1757 for (byte[] family : families) {
1758 HColumnDescriptor hcd = new HColumnDescriptor(family)
1759 .setMaxVersions(numVersions)
1760 .setBlocksize(blockSize);
1761 desc.addFamily(hcd);
1762 }
1763 getHBaseAdmin().createTable(desc);
1764
1765 waitUntilAllRegionsAssigned(tableName);
1766 return new HTable(new Configuration(getConfiguration()), tableName);
1767 }
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777 public HTable createTable(byte[] tableName, byte[][] families,
1778 int[] numVersions)
1779 throws IOException {
1780 return createTable(TableName.valueOf(tableName), families, numVersions);
1781 }
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791 public HTable createTable(TableName tableName, byte[][] families,
1792 int[] numVersions)
1793 throws IOException {
1794 HTableDescriptor desc = new HTableDescriptor(tableName);
1795 int i = 0;
1796 for (byte[] family : families) {
1797 HColumnDescriptor hcd = new HColumnDescriptor(family)
1798 .setMaxVersions(numVersions[i]);
1799 desc.addFamily(hcd);
1800 i++;
1801 }
1802 getHBaseAdmin().createTable(desc);
1803
1804 waitUntilAllRegionsAssigned(tableName);
1805 return new HTable(new Configuration(getConfiguration()), tableName);
1806 }
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816 public HTable createTable(byte[] tableName, byte[] family, byte[][] splitRows)
1817 throws IOException{
1818 return createTable(TableName.valueOf(tableName), family, splitRows);
1819 }
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829 public HTable createTable(TableName tableName, byte[] family, byte[][] splitRows)
1830 throws IOException {
1831 HTableDescriptor desc = new HTableDescriptor(tableName);
1832 HColumnDescriptor hcd = new HColumnDescriptor(family);
1833 desc.addFamily(hcd);
1834 getHBaseAdmin().createTable(desc, splitRows);
1835
1836 waitUntilAllRegionsAssigned(tableName);
1837 return new HTable(getConfiguration(), tableName);
1838 }
1839
1840
1841
1842
1843
1844
1845
1846
1847 public HTable createMultiRegionTable(TableName tableName, byte[] family) throws IOException {
1848 return createTable(tableName, family, KEYS_FOR_HBA_CREATE_TABLE);
1849 }
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859 public HTable createTable(byte[] tableName, byte[][] families, byte[][] splitRows)
1860 throws IOException {
1861 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
1862 for(byte[] family:families) {
1863 HColumnDescriptor hcd = new HColumnDescriptor(family);
1864 desc.addFamily(hcd);
1865 }
1866 getHBaseAdmin().createTable(desc, splitRows);
1867
1868 waitUntilAllRegionsAssigned(desc.getTableName());
1869 return new HTable(getConfiguration(), desc.getTableName());
1870 }
1871
1872
1873
1874
1875 public static WAL createWal(final Configuration conf, final Path rootDir, final Path walRootDir, final HRegionInfo hri)
1876 throws IOException {
1877
1878
1879 Configuration confForWAL = new Configuration(conf);
1880 confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
1881 FSUtils.setWALRootDir(confForWAL, walRootDir);
1882 return (new WALFactory(confForWAL,
1883 Collections.<WALActionsListener>singletonList(new MetricsWAL()),
1884 "hregion-" + RandomStringUtils.randomNumeric(8))).
1885 getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace());
1886 }
1887
1888
1889
1890
1891
1892 public static HRegion createRegionAndWAL(final HRegionInfo info, final Path rootDir,
1893 final Path walRootDir, final Configuration conf, final HTableDescriptor htd) throws IOException {
1894 return createRegionAndWAL(info, rootDir, walRootDir, conf, htd, true);
1895 }
1896
1897
1898
1899
1900
1901 public static HRegion createRegionAndWAL(final HRegionInfo info, final Path rootDir,
1902 final Path walRootDir, final Configuration conf, final HTableDescriptor htd, boolean initialize)
1903 throws IOException {
1904 WAL wal = createWal(conf, rootDir, walRootDir, info);
1905 return HRegion.createHRegion(info, rootDir, conf, htd, wal, initialize);
1906 }
1907
1908
1909
1910
1911 public static void closeRegionAndWAL(final Region r) throws IOException {
1912 closeRegionAndWAL((HRegion)r);
1913 }
1914
1915
1916
1917
1918 public static void closeRegionAndWAL(final HRegion r) throws IOException {
1919 if (r == null) return;
1920 r.close();
1921 if (r.getWAL() == null) return;
1922 r.getWAL().close();
1923 }
1924
1925
1926
1927
1928 @SuppressWarnings("serial")
1929 public static void modifyTableSync(Admin admin, HTableDescriptor desc)
1930 throws IOException, InterruptedException {
1931 admin.modifyTable(desc.getTableName(), desc);
1932 Pair<Integer, Integer> status = new Pair<Integer, Integer>() {{
1933 setFirst(0);
1934 setSecond(0);
1935 }};
1936 int i = 0;
1937 do {
1938 status = admin.getAlterStatus(desc.getTableName());
1939 if (status.getSecond() != 0) {
1940 LOG.debug(status.getSecond() - status.getFirst() + "/" + status.getSecond()
1941 + " regions updated.");
1942 Thread.sleep(1 * 1000l);
1943 } else {
1944 LOG.debug("All regions updated.");
1945 break;
1946 }
1947 } while (status.getFirst() != 0 && i++ < 500);
1948 if (status.getFirst() != 0) {
1949 throw new IOException("Failed to update all regions even after 500 seconds.");
1950 }
1951 }
1952
1953
1954
1955
1956 public static void setReplicas(Admin admin, TableName table, int replicaCount)
1957 throws IOException, InterruptedException {
1958 admin.disableTable(table);
1959 HTableDescriptor desc = admin.getTableDescriptor(table);
1960 desc.setRegionReplication(replicaCount);
1961 admin.modifyTable(desc.getTableName(), desc);
1962 admin.enableTable(table);
1963 }
1964
1965
1966
1967
1968
1969 public void deleteTable(String tableName) throws IOException {
1970 deleteTable(TableName.valueOf(tableName));
1971 }
1972
1973
1974
1975
1976
1977 public void deleteTable(byte[] tableName) throws IOException {
1978 deleteTable(TableName.valueOf(tableName));
1979 }
1980
1981
1982
1983
1984
1985 public void deleteTable(TableName tableName) throws IOException {
1986 try {
1987 getHBaseAdmin().disableTable(tableName);
1988 } catch (TableNotEnabledException e) {
1989 LOG.debug("Table: " + tableName + " already disabled, so just deleting it.");
1990 }
1991 getHBaseAdmin().deleteTable(tableName);
1992 }
1993
1994
1995
1996
1997
1998 public void deleteTableIfAny(TableName tableName) throws IOException {
1999 try {
2000 deleteTable(tableName);
2001 } catch (TableNotFoundException e) {
2002
2003 }
2004 }
2005
2006
2007
2008
2009
2010 public final static byte [] fam1 = Bytes.toBytes("colfamily11");
2011 public final static byte [] fam2 = Bytes.toBytes("colfamily21");
2012 public final static byte [] fam3 = Bytes.toBytes("colfamily31");
2013 public static final byte[][] COLUMNS = {fam1, fam2, fam3};
2014 private static final int MAXVERSIONS = 3;
2015
2016 public static final char FIRST_CHAR = 'a';
2017 public static final char LAST_CHAR = 'z';
2018 public static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
2019 public static final String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
2020
2021
2022
2023
2024
2025
2026
2027
2028 public HTableDescriptor createTableDescriptor(final String name,
2029 final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) {
2030 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
2031 for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
2032 htd.addFamily(new HColumnDescriptor(cfName)
2033 .setMinVersions(minVersions)
2034 .setMaxVersions(versions)
2035 .setKeepDeletedCells(keepDeleted)
2036 .setBlockCacheEnabled(false)
2037 .setTimeToLive(ttl)
2038 );
2039 }
2040 return htd;
2041 }
2042
2043
2044
2045
2046
2047
2048
2049 public HTableDescriptor createTableDescriptor(final String name) {
2050 return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
2051 MAXVERSIONS, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
2052 }
2053
2054
2055
2056
2057
2058 public HRegion createHRegion(
2059 final HRegionInfo info,
2060 final Path rootDir,
2061 final Configuration conf,
2062 final HTableDescriptor htd) throws IOException {
2063 return HRegion.createHRegion(info, rootDir, conf, htd);
2064 }
2065
2066 public HTableDescriptor createTableDescriptor(final TableName tableName,
2067 byte[] family) {
2068 return createTableDescriptor(tableName, new byte[][] {family}, 1);
2069 }
2070
2071 public HTableDescriptor createTableDescriptor(final TableName tableName,
2072 byte[][] families, int maxVersions) {
2073 HTableDescriptor desc = new HTableDescriptor(tableName);
2074 for (byte[] family : families) {
2075 HColumnDescriptor hcd = new HColumnDescriptor(family)
2076 .setMaxVersions(maxVersions);
2077 desc.addFamily(hcd);
2078 }
2079 return desc;
2080 }
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090 public HRegion createLocalHRegion(HTableDescriptor desc, byte [] startKey,
2091 byte [] endKey)
2092 throws IOException {
2093 HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
2094 return createLocalHRegion(hri, desc);
2095 }
2096
2097
2098
2099
2100
2101
2102
2103
2104 public HRegion createLocalHRegion(HRegionInfo info, HTableDescriptor desc) throws IOException {
2105 return HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), desc);
2106 }
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116 public HRegion createLocalHRegion(HRegionInfo info, HTableDescriptor desc, WAL wal)
2117 throws IOException {
2118 return HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), desc, wal);
2119 }
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133 public HRegion createLocalHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
2134 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
2135 WAL wal, byte[]... families) throws IOException {
2136 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
2137 htd.setReadOnly(isReadOnly);
2138 for (byte[] family : families) {
2139 HColumnDescriptor hcd = new HColumnDescriptor(family);
2140
2141 hcd.setMaxVersions(Integer.MAX_VALUE);
2142 htd.addFamily(hcd);
2143 }
2144 htd.setDurability(durability);
2145 HRegionInfo info = new HRegionInfo(htd.getTableName(), startKey, stopKey, false);
2146 return createLocalHRegion(info, htd, wal);
2147 }
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158 public HTable deleteTableData(byte[] tableName) throws IOException {
2159 return deleteTableData(TableName.valueOf(tableName));
2160 }
2161
2162
2163
2164
2165
2166
2167
2168
2169 public HTable deleteTableData(TableName tableName) throws IOException {
2170 HTable table = new HTable(getConfiguration(), tableName);
2171 Scan scan = new Scan();
2172 ResultScanner resScan = table.getScanner(scan);
2173 for(Result res : resScan) {
2174 Delete del = new Delete(res.getRow());
2175 table.delete(del);
2176 }
2177 resScan = table.getScanner(scan);
2178 resScan.close();
2179 return table;
2180 }
2181
2182
2183
2184
2185
2186
2187
2188
2189 public HTable truncateTable(final TableName tableName, final boolean preserveRegions)
2190 throws IOException {
2191 Admin admin = getHBaseAdmin();
2192 if (!admin.isTableDisabled(tableName)) {
2193 admin.disableTable(tableName);
2194 }
2195 admin.truncateTable(tableName, preserveRegions);
2196 return new HTable(getConfiguration(), tableName);
2197 }
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208 public HTable truncateTable(final TableName tableName) throws IOException {
2209 return truncateTable(tableName, false);
2210 }
2211
2212
2213
2214
2215
2216
2217
2218
2219 public HTable truncateTable(final byte[] tableName, final boolean preserveRegions)
2220 throws IOException {
2221 return truncateTable(TableName.valueOf(tableName), preserveRegions);
2222 }
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233 public HTable truncateTable(final byte[] tableName) throws IOException {
2234 return truncateTable(tableName, false);
2235 }
2236
2237
2238
2239
2240
2241
2242
2243
2244 public int loadTable(final Table t, final byte[] f) throws IOException {
2245 return loadTable(t, new byte[][] {f});
2246 }
2247
2248
2249
2250
2251
2252
2253
2254
2255 public int loadTable(final Table t, final byte[] f, boolean writeToWAL) throws IOException {
2256 return loadTable(t, new byte[][] {f}, null, writeToWAL);
2257 }
2258
2259
2260
2261
2262
2263
2264
2265
2266 public int loadTable(final Table t, final byte[][] f) throws IOException {
2267 return loadTable(t, f, null);
2268 }
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278 public int loadTable(final Table t, final byte[][] f, byte[] value) throws IOException {
2279 return loadTable(t, f, value, true);
2280 }
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290 public int loadTable(final Table t, final byte[][] f, byte[] value, boolean writeToWAL) throws IOException {
2291 List<Put> puts = new ArrayList<>();
2292 for (byte[] row : HBaseTestingUtility.ROWS) {
2293 Put put = new Put(row);
2294 put.setDurability(writeToWAL ? Durability.USE_DEFAULT : Durability.SKIP_WAL);
2295 for (int i = 0; i < f.length; i++) {
2296 put.add(f[i], f[i], value != null ? value : row);
2297 }
2298 puts.add(put);
2299 }
2300 t.put(puts);
2301 return puts.size();
2302 }
2303
2304
2305
2306
2307 public static class SeenRowTracker {
2308 int dim = 'z' - 'a' + 1;
2309 int[][][] seenRows = new int[dim][dim][dim];
2310 byte[] startRow;
2311 byte[] stopRow;
2312
2313 public SeenRowTracker(byte[] startRow, byte[] stopRow) {
2314 this.startRow = startRow;
2315 this.stopRow = stopRow;
2316 }
2317
2318 void reset() {
2319 for (byte[] row : ROWS) {
2320 seenRows[i(row[0])][i(row[1])][i(row[2])] = 0;
2321 }
2322 }
2323
2324 int i(byte b) {
2325 return b - 'a';
2326 }
2327
2328 public void addRow(byte[] row) {
2329 seenRows[i(row[0])][i(row[1])][i(row[2])]++;
2330 }
2331
2332
2333
2334
2335 public void validate() {
2336 for (byte b1 = 'a'; b1 <= 'z'; b1++) {
2337 for (byte b2 = 'a'; b2 <= 'z'; b2++) {
2338 for (byte b3 = 'a'; b3 <= 'z'; b3++) {
2339 int count = seenRows[i(b1)][i(b2)][i(b3)];
2340 int expectedCount = 0;
2341 if (Bytes.compareTo(new byte[] {b1,b2,b3}, startRow) >= 0
2342 && Bytes.compareTo(new byte[] {b1,b2,b3}, stopRow) < 0) {
2343 expectedCount = 1;
2344 }
2345 if (count != expectedCount) {
2346 String row = new String(new byte[] {b1,b2,b3});
2347 throw new RuntimeException("Row:" + row + " has a seen count of " + count + " instead of " + expectedCount);
2348 }
2349 }
2350 }
2351 }
2352 }
2353 }
2354
2355 public int loadRegion(final HRegion r, final byte[] f) throws IOException {
2356 return loadRegion(r, f, false);
2357 }
2358
2359 public int loadRegion(final Region r, final byte[] f) throws IOException {
2360 return loadRegion((HRegion)r, f);
2361 }
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371 public int loadRegion(final HRegion r, final byte[] f, final boolean flush)
2372 throws IOException {
2373 byte[] k = new byte[3];
2374 int rowCount = 0;
2375 for (byte b1 = 'a'; b1 <= 'z'; b1++) {
2376 for (byte b2 = 'a'; b2 <= 'z'; b2++) {
2377 for (byte b3 = 'a'; b3 <= 'z'; b3++) {
2378 k[0] = b1;
2379 k[1] = b2;
2380 k[2] = b3;
2381 Put put = new Put(k);
2382 put.setDurability(Durability.SKIP_WAL);
2383 put.add(f, null, k);
2384 if (r.getWAL() == null) {
2385 put.setDurability(Durability.SKIP_WAL);
2386 }
2387 int preRowCount = rowCount;
2388 int pause = 10;
2389 int maxPause = 1000;
2390 while (rowCount == preRowCount) {
2391 try {
2392 r.put(put);
2393 rowCount++;
2394 } catch (RegionTooBusyException e) {
2395 pause = (pause * 2 >= maxPause) ? maxPause : pause * 2;
2396 Threads.sleep(pause);
2397 }
2398 }
2399 }
2400 }
2401 if (flush) {
2402 r.flush(true);
2403 }
2404 }
2405 return rowCount;
2406 }
2407
2408 public void loadNumericRows(final Table t, final byte[] f, int startRow, int endRow)
2409 throws IOException {
2410 for (int i = startRow; i < endRow; i++) {
2411 byte[] data = Bytes.toBytes(String.valueOf(i));
2412 Put put = new Put(data);
2413 put.add(f, null, data);
2414 t.put(put);
2415 }
2416 }
2417
2418 public void verifyNumericRows(Table table, final byte[] f, int startRow, int endRow,
2419 int replicaId)
2420 throws IOException {
2421 for (int i = startRow; i < endRow; i++) {
2422 String failMsg = "Failed verification of row :" + i;
2423 byte[] data = Bytes.toBytes(String.valueOf(i));
2424 Get get = new Get(data);
2425 get.setReplicaId(replicaId);
2426 get.setConsistency(Consistency.TIMELINE);
2427 Result result = table.get(get);
2428 assertTrue(failMsg, result.containsColumn(f, null));
2429 assertEquals(failMsg, result.getColumnCells(f, null).size(), 1);
2430 Cell cell = result.getColumnLatestCell(f, null);
2431 assertTrue(failMsg,
2432 Bytes.equals(data, 0, data.length, cell.getValueArray(), cell.getValueOffset(),
2433 cell.getValueLength()));
2434 }
2435 }
2436
2437 public void verifyNumericRows(Region region, final byte[] f, int startRow, int endRow)
2438 throws IOException {
2439 verifyNumericRows((HRegion)region, f, startRow, endRow);
2440 }
2441
2442 public void verifyNumericRows(HRegion region, final byte[] f, int startRow, int endRow)
2443 throws IOException {
2444 verifyNumericRows(region, f, startRow, endRow, true);
2445 }
2446
2447 public void verifyNumericRows(Region region, final byte[] f, int startRow, int endRow,
2448 final boolean present) throws IOException {
2449 verifyNumericRows((HRegion)region, f, startRow, endRow, present);
2450 }
2451
2452 public void verifyNumericRows(HRegion region, final byte[] f, int startRow, int endRow,
2453 final boolean present) throws IOException {
2454 for (int i = startRow; i < endRow; i++) {
2455 String failMsg = "Failed verification of row :" + i;
2456 byte[] data = Bytes.toBytes(String.valueOf(i));
2457 Result result = region.get(new Get(data));
2458
2459 boolean hasResult = result != null && !result.isEmpty();
2460 assertEquals(failMsg + result, present, hasResult);
2461 if (!present) continue;
2462
2463 assertTrue(failMsg, result.containsColumn(f, null));
2464 assertEquals(failMsg, result.getColumnCells(f, null).size(), 1);
2465 Cell cell = result.getColumnLatestCell(f, null);
2466 assertTrue(failMsg,
2467 Bytes.equals(data, 0, data.length, cell.getValueArray(), cell.getValueOffset(),
2468 cell.getValueLength()));
2469 }
2470 }
2471
2472 public void deleteNumericRows(final HTable t, final byte[] f, int startRow, int endRow)
2473 throws IOException {
2474 for (int i = startRow; i < endRow; i++) {
2475 byte[] data = Bytes.toBytes(String.valueOf(i));
2476 Delete delete = new Delete(data);
2477 delete.deleteFamily(f);
2478 t.delete(delete);
2479 }
2480 }
2481
2482
2483
2484
2485 public int countRows(final Table table) throws IOException {
2486 return countRows(table, new Scan());
2487 }
2488
2489 public int countRows(final Table table, final Scan scan) throws IOException {
2490 try (ResultScanner results = table.getScanner(scan)) {
2491 int count = 0;
2492 while (results.next() != null) {
2493 count++;
2494 }
2495 return count;
2496 }
2497 }
2498
2499 public int countRows(final Table table, final byte[]... families) throws IOException {
2500 Scan scan = new Scan();
2501 for (byte[] family: families) {
2502 scan.addFamily(family);
2503 }
2504 return countRows(table, scan);
2505 }
2506
2507
2508
2509
2510 public int countRows(final TableName tableName) throws IOException {
2511 Table table = getConnection().getTable(tableName);
2512 try {
2513 return countRows(table);
2514 } finally {
2515 table.close();
2516 }
2517 }
2518
2519
2520
2521
2522 public String checksumRows(final Table table) throws Exception {
2523 Scan scan = new Scan();
2524 ResultScanner results = table.getScanner(scan);
2525 MessageDigest digest = MessageDigest.getInstance("MD5");
2526 for (Result res : results) {
2527 digest.update(res.getRow());
2528 }
2529 results.close();
2530 return digest.toString();
2531 }
2532
2533
2534 public static final byte[][] ROWS = new byte[(int) Math.pow('z' - 'a' + 1, 3)][3];
2535 static {
2536 int i = 0;
2537 for (byte b1 = 'a'; b1 <= 'z'; b1++) {
2538 for (byte b2 = 'a'; b2 <= 'z'; b2++) {
2539 for (byte b3 = 'a'; b3 <= 'z'; b3++) {
2540 ROWS[i][0] = b1;
2541 ROWS[i][1] = b2;
2542 ROWS[i][2] = b3;
2543 i++;
2544 }
2545 }
2546 }
2547 }
2548
2549 public static final byte[][] KEYS = {
2550 HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes("bbb"),
2551 Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
2552 Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
2553 Bytes.toBytes("iii"), Bytes.toBytes("jjj"), Bytes.toBytes("kkk"),
2554 Bytes.toBytes("lll"), Bytes.toBytes("mmm"), Bytes.toBytes("nnn"),
2555 Bytes.toBytes("ooo"), Bytes.toBytes("ppp"), Bytes.toBytes("qqq"),
2556 Bytes.toBytes("rrr"), Bytes.toBytes("sss"), Bytes.toBytes("ttt"),
2557 Bytes.toBytes("uuu"), Bytes.toBytes("vvv"), Bytes.toBytes("www"),
2558 Bytes.toBytes("xxx"), Bytes.toBytes("yyy")
2559 };
2560
2561 public static final byte[][] KEYS_FOR_HBA_CREATE_TABLE = {
2562 Bytes.toBytes("bbb"),
2563 Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
2564 Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
2565 Bytes.toBytes("iii"), Bytes.toBytes("jjj"), Bytes.toBytes("kkk"),
2566 Bytes.toBytes("lll"), Bytes.toBytes("mmm"), Bytes.toBytes("nnn"),
2567 Bytes.toBytes("ooo"), Bytes.toBytes("ppp"), Bytes.toBytes("qqq"),
2568 Bytes.toBytes("rrr"), Bytes.toBytes("sss"), Bytes.toBytes("ttt"),
2569 Bytes.toBytes("uuu"), Bytes.toBytes("vvv"), Bytes.toBytes("www"),
2570 Bytes.toBytes("xxx"), Bytes.toBytes("yyy"), Bytes.toBytes("zzz")
2571 };
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583 public List<HRegionInfo> createMultiRegionsInMeta(final Configuration conf,
2584 final HTableDescriptor htd, byte [][] startKeys)
2585 throws IOException {
2586 Table meta = new HTable(conf, TableName.META_TABLE_NAME);
2587 Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR);
2588 List<HRegionInfo> newRegions = new ArrayList<HRegionInfo>(startKeys.length);
2589
2590 for (int i = 0; i < startKeys.length; i++) {
2591 int j = (i + 1) % startKeys.length;
2592 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKeys[i],
2593 startKeys[j]);
2594 MetaTableAccessor.addRegionToMeta(meta, hri);
2595 newRegions.add(hri);
2596 }
2597
2598 meta.close();
2599 return newRegions;
2600 }
2601
2602
2603
2604
2605
2606
2607 public List<byte[]> getMetaTableRows() throws IOException {
2608
2609 Table t = new HTable(new Configuration(this.conf), TableName.META_TABLE_NAME);
2610 List<byte[]> rows = new ArrayList<byte[]>();
2611 ResultScanner s = t.getScanner(new Scan());
2612 for (Result result : s) {
2613 LOG.info("getMetaTableRows: row -> " +
2614 Bytes.toStringBinary(result.getRow()));
2615 rows.add(result.getRow());
2616 }
2617 s.close();
2618 t.close();
2619 return rows;
2620 }
2621
2622
2623
2624
2625
2626
2627 public List<byte[]> getMetaTableRows(TableName tableName) throws IOException {
2628
2629 Table t = new HTable(new Configuration(this.conf), TableName.META_TABLE_NAME);
2630 List<byte[]> rows = new ArrayList<byte[]>();
2631 ResultScanner s = t.getScanner(new Scan());
2632 for (Result result : s) {
2633 HRegionInfo info = HRegionInfo.getHRegionInfo(result);
2634 if (info == null) {
2635 LOG.error("No region info for row " + Bytes.toString(result.getRow()));
2636
2637 continue;
2638 }
2639
2640 if (info.getTable().equals(tableName)) {
2641 LOG.info("getMetaTableRows: row -> " +
2642 Bytes.toStringBinary(result.getRow()) + info);
2643 rows.add(result.getRow());
2644 }
2645 }
2646 s.close();
2647 t.close();
2648 return rows;
2649 }
2650
2651
2652
2653
2654
2655
2656 public HRegionServer getOtherRegionServer(HRegionServer rs) {
2657 for (JVMClusterUtil.RegionServerThread rst :
2658 getMiniHBaseCluster().getRegionServerThreads()) {
2659 if (!(rst.getRegionServer() == rs)) {
2660 return rst.getRegionServer();
2661 }
2662 }
2663 return null;
2664 }
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677 public HRegionServer getRSForFirstRegionInTable(TableName tableName)
2678 throws IOException, InterruptedException {
2679 List<byte[]> metaRows = getMetaTableRows(tableName);
2680 if (metaRows == null || metaRows.isEmpty()) {
2681 return null;
2682 }
2683 LOG.debug("Found " + metaRows.size() + " rows for table " +
2684 tableName);
2685 byte [] firstrow = metaRows.get(0);
2686 LOG.debug("FirstRow=" + Bytes.toString(firstrow));
2687 long pause = getConfiguration().getLong(HConstants.HBASE_CLIENT_PAUSE,
2688 HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
2689 int numRetries = getConfiguration().getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
2690 HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
2691 RetryCounter retrier = new RetryCounter(numRetries+1, (int)pause, TimeUnit.MICROSECONDS);
2692 while(retrier.shouldRetry()) {
2693 int index = getMiniHBaseCluster().getServerWith(firstrow);
2694 if (index != -1) {
2695 return getMiniHBaseCluster().getRegionServerThreads().get(index).getRegionServer();
2696 }
2697
2698 retrier.sleepUntilNextRetry();
2699 }
2700 return null;
2701 }
2702
2703
2704
2705
2706
2707
2708
2709 public MiniMRCluster startMiniMapReduceCluster() throws IOException {
2710 startMiniMapReduceCluster(2);
2711 return mrCluster;
2712 }
2713
2714
2715
2716
2717
2718 private void forceChangeTaskLogDir() {
2719 Field logDirField;
2720 try {
2721 logDirField = TaskLog.class.getDeclaredField("LOG_DIR");
2722 logDirField.setAccessible(true);
2723
2724 Field modifiersField = Field.class.getDeclaredField("modifiers");
2725 modifiersField.setAccessible(true);
2726 modifiersField.setInt(logDirField, logDirField.getModifiers() & ~Modifier.FINAL);
2727
2728 logDirField.set(null, new File(hadoopLogDir, "userlogs"));
2729 } catch (SecurityException e) {
2730 throw new RuntimeException(e);
2731 } catch (NoSuchFieldException e) {
2732
2733 throw new RuntimeException(e);
2734 } catch (IllegalArgumentException e) {
2735 throw new RuntimeException(e);
2736 } catch (IllegalAccessException e) {
2737 throw new RuntimeException(e);
2738 }
2739 }
2740
2741
2742
2743
2744
2745
2746
2747 private void startMiniMapReduceCluster(final int servers) throws IOException {
2748 if (mrCluster != null) {
2749 throw new IllegalStateException("MiniMRCluster is already running");
2750 }
2751 LOG.info("Starting mini mapreduce cluster...");
2752 setupClusterTestDir();
2753 createDirsAndSetProperties();
2754
2755 forceChangeTaskLogDir();
2756
2757
2758
2759
2760 conf.setFloat("yarn.nodemanager.vmem-pmem-ratio", 8.0f);
2761
2762
2763
2764 conf.setBoolean("mapreduce.map.speculative", false);
2765 conf.setBoolean("mapreduce.reduce.speculative", false);
2766
2767
2768
2769 mrCluster = new MiniMRCluster(servers,
2770 FS_URI != null ? FS_URI : FileSystem.get(conf).getUri().toString(), 1,
2771 null, null, new JobConf(this.conf));
2772 JobConf jobConf = MapreduceTestingShim.getJobConf(mrCluster);
2773 if (jobConf == null) {
2774 jobConf = mrCluster.createJobConf();
2775 }
2776
2777 jobConf.set("mapreduce.cluster.local.dir",
2778 conf.get("mapreduce.cluster.local.dir"));
2779 LOG.info("Mini mapreduce cluster started");
2780
2781
2782
2783
2784 conf.set("mapreduce.jobtracker.address", jobConf.get("mapreduce.jobtracker.address"));
2785
2786 conf.set("mapreduce.framework.name", "yarn");
2787 conf.setBoolean("yarn.is.minicluster", true);
2788 String rmAddress = jobConf.get("yarn.resourcemanager.address");
2789 if (rmAddress != null) {
2790 conf.set("yarn.resourcemanager.address", rmAddress);
2791 }
2792 String historyAddress = jobConf.get("mapreduce.jobhistory.address");
2793 if (historyAddress != null) {
2794 conf.set("mapreduce.jobhistory.address", historyAddress);
2795 }
2796 String schedulerAddress =
2797 jobConf.get("yarn.resourcemanager.scheduler.address");
2798 if (schedulerAddress != null) {
2799 conf.set("yarn.resourcemanager.scheduler.address", schedulerAddress);
2800 }
2801 String mrJobHistoryWebappAddress =
2802 jobConf.get("mapreduce.jobhistory.webapp.address");
2803 if (mrJobHistoryWebappAddress != null) {
2804 conf.set("mapreduce.jobhistory.webapp.address", mrJobHistoryWebappAddress);
2805 }
2806 String yarnRMWebappAddress =
2807 jobConf.get("yarn.resourcemanager.webapp.address");
2808 if (yarnRMWebappAddress != null) {
2809 conf.set("yarn.resourcemanager.webapp.address", yarnRMWebappAddress);
2810 }
2811 }
2812
2813
2814
2815
2816 public void shutdownMiniMapReduceCluster() {
2817 if (mrCluster != null) {
2818 LOG.info("Stopping mini mapreduce cluster...");
2819 mrCluster.shutdown();
2820 mrCluster = null;
2821 LOG.info("Mini mapreduce cluster stopped");
2822 }
2823
2824 conf.set("mapreduce.jobtracker.address", "local");
2825 }
2826
2827
2828
2829
2830 public RegionServerServices createMockRegionServerService() throws IOException {
2831 return createMockRegionServerService((ServerName)null);
2832 }
2833
2834
2835
2836
2837
2838 public RegionServerServices createMockRegionServerService(RpcServerInterface rpc) throws IOException {
2839 final MockRegionServerServices rss = new MockRegionServerServices(getZooKeeperWatcher());
2840 rss.setFileSystem(getTestFileSystem());
2841 rss.setRpcServer(rpc);
2842 return rss;
2843 }
2844
2845
2846
2847
2848
2849 public RegionServerServices createMockRegionServerService(ServerName name) throws IOException {
2850 final MockRegionServerServices rss = new MockRegionServerServices(getZooKeeperWatcher(), name);
2851 rss.setFileSystem(getTestFileSystem());
2852 return rss;
2853 }
2854
2855
2856
2857
2858
2859
2860 public void enableDebug(Class<?> clazz) {
2861 Log l = LogFactory.getLog(clazz);
2862 if (l instanceof Log4JLogger) {
2863 ((Log4JLogger) l).getLogger().setLevel(org.apache.log4j.Level.DEBUG);
2864 } else if (l instanceof Jdk14Logger) {
2865 ((Jdk14Logger) l).getLogger().setLevel(java.util.logging.Level.ALL);
2866 }
2867 }
2868
2869
2870
2871
2872
2873 public void expireMasterSession() throws Exception {
2874 HMaster master = getMiniHBaseCluster().getMaster();
2875 expireSession(master.getZooKeeper(), false);
2876 }
2877
2878
2879
2880
2881
2882
2883 public void expireRegionServerSession(int index) throws Exception {
2884 HRegionServer rs = getMiniHBaseCluster().getRegionServer(index);
2885 expireSession(rs.getZooKeeper(), false);
2886 decrementMinRegionServerCount();
2887 }
2888
2889 private void decrementMinRegionServerCount() {
2890
2891
2892 decrementMinRegionServerCount(getConfiguration());
2893
2894
2895 for (MasterThread master : getHBaseCluster().getMasterThreads()) {
2896 decrementMinRegionServerCount(master.getMaster().getConfiguration());
2897 }
2898 }
2899
2900 private void decrementMinRegionServerCount(Configuration conf) {
2901 int currentCount = conf.getInt(
2902 ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1);
2903 if (currentCount != -1) {
2904 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
2905 Math.max(currentCount - 1, 1));
2906 }
2907 }
2908
2909 public void expireSession(ZooKeeperWatcher nodeZK) throws Exception {
2910 expireSession(nodeZK, false);
2911 }
2912
2913 @Deprecated
2914 public void expireSession(ZooKeeperWatcher nodeZK, Server server)
2915 throws Exception {
2916 expireSession(nodeZK, false);
2917 }
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930 public void expireSession(ZooKeeperWatcher nodeZK, boolean checkStatus)
2931 throws Exception {
2932 Configuration c = new Configuration(this.conf);
2933 String quorumServers = ZKConfig.getZKQuorumServersString(c);
2934 ZooKeeper zk = nodeZK.getRecoverableZooKeeper().getZooKeeper();
2935 byte[] password = zk.getSessionPasswd();
2936 long sessionID = zk.getSessionId();
2937
2938
2939
2940
2941
2942
2943
2944
2945 ZooKeeper monitor = new ZooKeeper(quorumServers,
2946 1000, new org.apache.zookeeper.Watcher(){
2947 @Override
2948 public void process(WatchedEvent watchedEvent) {
2949 LOG.info("Monitor ZKW received event="+watchedEvent);
2950 }
2951 } , sessionID, password);
2952
2953
2954 ZooKeeper newZK = new ZooKeeper(quorumServers,
2955 1000, EmptyWatcher.instance, sessionID, password);
2956
2957
2958
2959 long start = System.currentTimeMillis();
2960 while (newZK.getState() != States.CONNECTED
2961 && System.currentTimeMillis() - start < 1000) {
2962 Thread.sleep(1);
2963 }
2964 newZK.close();
2965 LOG.info("ZK Closed Session 0x" + Long.toHexString(sessionID));
2966
2967
2968 monitor.close();
2969
2970 if (checkStatus) {
2971 new HTable(new Configuration(conf), TableName.META_TABLE_NAME).close();
2972 }
2973 }
2974
2975
2976
2977
2978
2979
2980
2981 public MiniHBaseCluster getHBaseCluster() {
2982 return getMiniHBaseCluster();
2983 }
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993 public HBaseCluster getHBaseClusterInterface() {
2994
2995
2996 return hbaseCluster;
2997 }
2998
2999
3000
3001
3002
3003
3004
3005 public Connection getConnection() throws IOException {
3006 if (this.connection == null) {
3007 this.connection = ConnectionFactory.createConnection(this.conf);
3008 }
3009 return this.connection;
3010 }
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021 public synchronized HBaseAdmin getHBaseAdmin()
3022 throws IOException {
3023 if (hbaseAdmin == null){
3024 this.hbaseAdmin = new HBaseAdminForTests(getConnection());
3025 }
3026 return hbaseAdmin;
3027 }
3028
3029 private HBaseAdminForTests hbaseAdmin = null;
3030 private static class HBaseAdminForTests extends HBaseAdmin {
3031 public HBaseAdminForTests(Connection connection) throws MasterNotRunningException,
3032 ZooKeeperConnectionException, IOException {
3033 super(connection);
3034 }
3035
3036 @Override
3037 public synchronized void close() throws IOException {
3038 LOG.warn("close() called on HBaseAdmin instance returned from " +
3039 "HBaseTestingUtility.getHBaseAdmin()");
3040 }
3041
3042 private synchronized void close0() throws IOException {
3043 super.close();
3044 }
3045 }
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056 public synchronized ZooKeeperWatcher getZooKeeperWatcher()
3057 throws IOException {
3058 if (zooKeeperWatcher == null) {
3059 zooKeeperWatcher = new ZooKeeperWatcher(conf, "testing utility",
3060 new Abortable() {
3061 @Override public void abort(String why, Throwable e) {
3062 throw new RuntimeException("Unexpected abort in HBaseTestingUtility:"+why, e);
3063 }
3064 @Override public boolean isAborted() {return false;}
3065 });
3066 }
3067 return zooKeeperWatcher;
3068 }
3069 private ZooKeeperWatcher zooKeeperWatcher;
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079 public void closeRegion(String regionName) throws IOException {
3080 closeRegion(Bytes.toBytes(regionName));
3081 }
3082
3083
3084
3085
3086
3087
3088
3089 public void closeRegion(byte[] regionName) throws IOException {
3090 getHBaseAdmin().closeRegion(regionName, null);
3091 }
3092
3093
3094
3095
3096
3097
3098
3099
3100 public void closeRegionByRow(String row, RegionLocator table) throws IOException {
3101 closeRegionByRow(Bytes.toBytes(row), table);
3102 }
3103
3104
3105
3106
3107
3108
3109
3110
3111 public void closeRegionByRow(byte[] row, RegionLocator table) throws IOException {
3112 HRegionLocation hrl = table.getRegionLocation(row);
3113 closeRegion(hrl.getRegionInfo().getRegionName());
3114 }
3115
3116
3117
3118
3119
3120
3121
3122
3123 public HRegion getSplittableRegion(TableName tableName, int maxAttempts) {
3124 List<HRegion> regions = getHBaseCluster().getRegions(tableName);
3125 int regCount = regions.size();
3126 Set<Integer> attempted = new HashSet<Integer>();
3127 int idx;
3128 int attempts = 0;
3129 do {
3130 regions = getHBaseCluster().getRegions(tableName);
3131 if (regCount != regions.size()) {
3132
3133 attempted.clear();
3134 }
3135 regCount = regions.size();
3136
3137
3138 if (regCount > 0) {
3139 idx = random.nextInt(regCount);
3140
3141 if (attempted.contains(idx))
3142 continue;
3143 try {
3144 regions.get(idx).checkSplit();
3145 return regions.get(idx);
3146 } catch (Exception ex) {
3147 LOG.warn("Caught exception", ex);
3148 attempted.add(idx);
3149 }
3150 }
3151 attempts++;
3152 } while (maxAttempts == -1 || attempts < maxAttempts);
3153 return null;
3154 }
3155
3156 public MiniZooKeeperCluster getZkCluster() {
3157 return zkCluster;
3158 }
3159
3160 public void setZkCluster(MiniZooKeeperCluster zkCluster) {
3161 this.passedZkCluster = true;
3162 this.zkCluster = zkCluster;
3163 conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkCluster.getClientPort());
3164 }
3165
3166 public MiniDFSCluster getDFSCluster() {
3167 return dfsCluster;
3168 }
3169
3170 public void setDFSCluster(MiniDFSCluster cluster) throws IllegalStateException, IOException {
3171 setDFSCluster(cluster, true);
3172 }
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182 public void setDFSCluster(MiniDFSCluster cluster, boolean requireDown)
3183 throws IllegalStateException, IOException {
3184 if (dfsCluster != null && requireDown && dfsCluster.isClusterUp()) {
3185 throw new IllegalStateException("DFSCluster is already running! Shut it down first.");
3186 }
3187 this.dfsCluster = cluster;
3188 this.setFs();
3189 }
3190
3191 public FileSystem getTestFileSystem() throws IOException {
3192 return HFileSystem.get(conf);
3193 }
3194
3195
3196
3197
3198
3199
3200
3201
3202 public void waitTableAvailable(TableName table)
3203 throws InterruptedException, IOException {
3204 waitTableAvailable(table.getName(), 30000);
3205 }
3206
3207 public void waitTableAvailable(TableName table, long timeoutMillis)
3208 throws InterruptedException, IOException {
3209 waitFor(timeoutMillis, predicateTableAvailable(table));
3210 }
3211
3212 public String explainTableAvailability(TableName tableName) throws IOException {
3213 String msg = explainTableState(tableName) + ",";
3214 if (getHBaseCluster().getMaster().isAlive()) {
3215 Map<HRegionInfo, ServerName> assignments =
3216 getHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
3217 .getRegionAssignments();
3218 final List<Pair<HRegionInfo, ServerName>> metaLocations =
3219 MetaTableAccessor
3220 .getTableRegionsAndLocations(getZooKeeperWatcher(), connection, tableName);
3221 for (Pair<HRegionInfo, ServerName> metaLocation : metaLocations) {
3222 HRegionInfo hri = metaLocation.getFirst();
3223 ServerName sn = metaLocation.getSecond();
3224 if (!assignments.containsKey(hri)) {
3225 msg += ", region " + hri
3226 + " not assigned, but found in meta, it expected to be on " + sn;
3227
3228 } else if (sn == null) {
3229 msg += ", region " + hri
3230 + " assigned, but has no server in meta";
3231 } else if (!sn.equals(assignments.get(hri))) {
3232 msg += ", region " + hri
3233 + " assigned, but has different servers in meta and AM ( " +
3234 sn + " <> " + assignments.get(hri);
3235 }
3236 }
3237 }
3238 return msg;
3239 }
3240
3241 public String explainTableState(TableName tableName) throws IOException {
3242 try {
3243 if (getHBaseAdmin().isTableEnabled(tableName))
3244 return "table enabled in zk";
3245 else if (getHBaseAdmin().isTableDisabled(tableName))
3246 return "table disabled in zk";
3247 else
3248 return "table in uknown state";
3249 } catch (TableNotFoundException e) {
3250 return "table not exists";
3251 }
3252 }
3253
3254
3255
3256
3257
3258
3259
3260
3261 public void waitTableAvailable(byte[] table, long timeoutMillis)
3262 throws InterruptedException, IOException {
3263 waitTableAvailable(getHBaseAdmin(), table, timeoutMillis);
3264 }
3265
3266 public void waitTableAvailable(Admin admin, byte[] table, long timeoutMillis)
3267 throws InterruptedException, IOException {
3268 long startWait = System.currentTimeMillis();
3269 while (!admin.isTableAvailable(TableName.valueOf(table))) {
3270 assertTrue("Timed out waiting for table to become available " +
3271 Bytes.toStringBinary(table),
3272 System.currentTimeMillis() - startWait < timeoutMillis);
3273 Thread.sleep(200);
3274 }
3275 }
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286 public void waitTableEnabled(TableName table)
3287 throws InterruptedException, IOException {
3288 waitTableEnabled(table, 30000);
3289 }
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300 public void waitTableEnabled(byte[] table, long timeoutMillis)
3301 throws InterruptedException, IOException {
3302 waitTableEnabled(TableName.valueOf(table), timeoutMillis);
3303 }
3304
3305 public void waitTableEnabled(TableName table, long timeoutMillis)
3306 throws IOException {
3307 waitFor(timeoutMillis, predicateTableEnabled(table));
3308 }
3309
3310
3311
3312
3313
3314
3315
3316
3317 public void waitTableDisabled(byte[] table)
3318 throws InterruptedException, IOException {
3319 waitTableDisabled(getHBaseAdmin(), table, 30000);
3320 }
3321
3322 public void waitTableDisabled(Admin admin, byte[] table)
3323 throws InterruptedException, IOException {
3324 waitTableDisabled(admin, table, 30000);
3325 }
3326
3327
3328
3329
3330
3331
3332
3333
3334 public void waitTableDisabled(byte[] table, long timeoutMillis)
3335 throws InterruptedException, IOException {
3336 waitTableDisabled(getHBaseAdmin(), table, timeoutMillis);
3337 }
3338
3339 public void waitTableDisabled(Admin admin, byte[] table, long timeoutMillis)
3340 throws InterruptedException, IOException {
3341 TableName tableName = TableName.valueOf(table);
3342 long startWait = System.currentTimeMillis();
3343 while (!admin.isTableDisabled(tableName)) {
3344 assertTrue("Timed out waiting for table to become disabled " +
3345 Bytes.toStringBinary(table),
3346 System.currentTimeMillis() - startWait < timeoutMillis);
3347 Thread.sleep(200);
3348 }
3349 }
3350
3351
3352
3353
3354
3355
3356
3357
3358 public boolean ensureSomeRegionServersAvailable(final int num)
3359 throws IOException {
3360 boolean startedServer = false;
3361 MiniHBaseCluster hbaseCluster = getMiniHBaseCluster();
3362 for (int i=hbaseCluster.getLiveRegionServerThreads().size(); i<num; ++i) {
3363 LOG.info("Started new server=" + hbaseCluster.startRegionServer());
3364 startedServer = true;
3365 }
3366
3367 return startedServer;
3368 }
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379 public boolean ensureSomeNonStoppedRegionServersAvailable(final int num)
3380 throws IOException {
3381 boolean startedServer = ensureSomeRegionServersAvailable(num);
3382
3383 int nonStoppedServers = 0;
3384 for (JVMClusterUtil.RegionServerThread rst :
3385 getMiniHBaseCluster().getRegionServerThreads()) {
3386
3387 HRegionServer hrs = rst.getRegionServer();
3388 if (hrs.isStopping() || hrs.isStopped()) {
3389 LOG.info("A region server is stopped or stopping:"+hrs);
3390 } else {
3391 nonStoppedServers++;
3392 }
3393 }
3394 for (int i=nonStoppedServers; i<num; ++i) {
3395 LOG.info("Started new server=" + getMiniHBaseCluster().startRegionServer());
3396 startedServer = true;
3397 }
3398 return startedServer;
3399 }
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411 public static User getDifferentUser(final Configuration c,
3412 final String differentiatingSuffix)
3413 throws IOException {
3414 FileSystem currentfs = FileSystem.get(c);
3415 if (!(currentfs instanceof DistributedFileSystem) || User.isHBaseSecurityEnabled(c)) {
3416 return User.getCurrent();
3417 }
3418
3419
3420 String username = User.getCurrent().getName() +
3421 differentiatingSuffix;
3422 User user = User.createUserForTesting(c, username,
3423 new String[]{"supergroup"});
3424 return user;
3425 }
3426
3427 public static NavigableSet<String> getAllOnlineRegions(MiniHBaseCluster cluster)
3428 throws IOException {
3429 NavigableSet<String> online = new TreeSet<String>();
3430 for (RegionServerThread rst : cluster.getLiveRegionServerThreads()) {
3431 try {
3432 for (HRegionInfo region :
3433 ProtobufUtil.getOnlineRegions(rst.getRegionServer().getRSRpcServices())) {
3434 online.add(region.getRegionNameAsString());
3435 }
3436 } catch (RegionServerStoppedException e) {
3437
3438 }
3439 }
3440 for (MasterThread mt : cluster.getLiveMasterThreads()) {
3441 try {
3442 for (HRegionInfo region :
3443 ProtobufUtil.getOnlineRegions(mt.getMaster().getRSRpcServices())) {
3444 online.add(region.getRegionNameAsString());
3445 }
3446 } catch (RegionServerStoppedException e) {
3447
3448 } catch (ServerNotRunningYetException e) {
3449
3450 }
3451 }
3452 return online;
3453 }
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468 public static void setMaxRecoveryErrorCount(final OutputStream stream,
3469 final int max) {
3470 try {
3471 Class<?> [] clazzes = DFSClient.class.getDeclaredClasses();
3472 for (Class<?> clazz: clazzes) {
3473 String className = clazz.getSimpleName();
3474 if (className.equals("DFSOutputStream")) {
3475 if (clazz.isInstance(stream)) {
3476 Field maxRecoveryErrorCountField =
3477 stream.getClass().getDeclaredField("maxRecoveryErrorCount");
3478 maxRecoveryErrorCountField.setAccessible(true);
3479 maxRecoveryErrorCountField.setInt(stream, max);
3480 break;
3481 }
3482 }
3483 }
3484 } catch (Exception e) {
3485 LOG.info("Could not set max recovery field", e);
3486 }
3487 }
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497 public boolean assignRegion(final HRegionInfo regionInfo)
3498 throws IOException, InterruptedException {
3499 final AssignmentManager am = getHBaseCluster().getMaster().getAssignmentManager();
3500 am.assign(regionInfo, true);
3501 return am.waitForAssignment(regionInfo);
3502 }
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512 public void moveRegionAndWait(HRegionInfo destRegion, ServerName destServer)
3513 throws InterruptedException, IOException {
3514 HMaster master = getMiniHBaseCluster().getMaster();
3515 getHBaseAdmin().move(destRegion.getEncodedNameAsBytes(),
3516 Bytes.toBytes(destServer.getServerName()));
3517 while (true) {
3518 ServerName serverName = master.getAssignmentManager().getRegionStates()
3519 .getRegionServerOfRegion(destRegion);
3520 if (serverName != null && serverName.equals(destServer)) {
3521 assertRegionOnServer(destRegion, serverName, 200);
3522 break;
3523 }
3524 Thread.sleep(10);
3525 }
3526 }
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537 public void waitUntilAllRegionsAssigned(final TableName tableName) throws IOException {
3538 waitUntilAllRegionsAssigned(
3539 tableName,
3540 this.conf.getLong("hbase.client.sync.wait.timeout.msec", 60000));
3541 }
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552 public void waitUntilAllRegionsAssigned(final TableName tableName, final long timeout)
3553 throws IOException {
3554 final Table meta = new HTable(getConfiguration(), TableName.META_TABLE_NAME);
3555 try {
3556 waitFor(timeout, 200, true, new Predicate<IOException>() {
3557 @Override
3558 public boolean evaluate() throws IOException {
3559 boolean allRegionsAssigned = true;
3560 Scan scan = new Scan();
3561 scan.addFamily(HConstants.CATALOG_FAMILY);
3562 ResultScanner s = meta.getScanner(scan);
3563 try {
3564 Result r;
3565 while ((r = s.next()) != null) {
3566 byte[] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
3567 HRegionInfo info = HRegionInfo.parseFromOrNull(b);
3568 if (info != null && info.getTable().equals(tableName)) {
3569 b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
3570 allRegionsAssigned &= (b != null);
3571 }
3572 }
3573 } finally {
3574 s.close();
3575 }
3576 return allRegionsAssigned;
3577 }
3578 });
3579 } finally {
3580 meta.close();
3581 }
3582
3583
3584 if (!getHBaseClusterInterface().isDistributedCluster()) {
3585
3586
3587 HMaster master = getHBaseCluster().getMaster();
3588 final RegionStates states = master.getAssignmentManager().getRegionStates();
3589 waitFor(timeout, 200, new ExplainingPredicate<IOException>() {
3590 @Override
3591 public String explainFailure() throws IOException {
3592 return explainTableAvailability(tableName);
3593 }
3594
3595 @Override
3596 public boolean evaluate() throws IOException {
3597 List<HRegionInfo> hris = states.getRegionsOfTable(tableName);
3598 return hris != null && !hris.isEmpty();
3599 }
3600 });
3601 }
3602 }
3603
3604
3605
3606
3607
3608 public static List<Cell> getFromStoreFile(HStore store,
3609 Get get) throws IOException {
3610 Scan scan = new Scan(get);
3611 InternalScanner scanner = (InternalScanner) store.getScanner(scan,
3612 scan.getFamilyMap().get(store.getFamily().getName()),
3613
3614
3615 0);
3616
3617 List<Cell> result = new ArrayList<Cell>();
3618 scanner.next(result);
3619 if (!result.isEmpty()) {
3620
3621 Cell kv = result.get(0);
3622 if (!CellUtil.matchingRow(kv, get.getRow())) {
3623 result.clear();
3624 }
3625 }
3626 scanner.close();
3627 return result;
3628 }
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638 public byte[][] getRegionSplitStartKeys(byte[] startKey, byte[] endKey, int numRegions){
3639 assertTrue(numRegions>3);
3640 byte [][] tmpSplitKeys = Bytes.split(startKey, endKey, numRegions - 3);
3641 byte [][] result = new byte[tmpSplitKeys.length+1][];
3642 System.arraycopy(tmpSplitKeys, 0, result, 1, tmpSplitKeys.length);
3643 result[0] = HConstants.EMPTY_BYTE_ARRAY;
3644 return result;
3645 }
3646
3647
3648
3649
3650
3651 public static List<Cell> getFromStoreFile(HStore store,
3652 byte [] row,
3653 NavigableSet<byte[]> columns
3654 ) throws IOException {
3655 Get get = new Get(row);
3656 Map<byte[], NavigableSet<byte[]>> s = get.getFamilyMap();
3657 s.put(store.getFamily().getName(), columns);
3658
3659 return getFromStoreFile(store,get);
3660 }
3661
3662
3663
3664
3665
3666 public static ZooKeeperWatcher getZooKeeperWatcher(
3667 HBaseTestingUtility TEST_UTIL) throws ZooKeeperConnectionException,
3668 IOException {
3669 ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
3670 "unittest", new Abortable() {
3671 boolean aborted = false;
3672
3673 @Override
3674 public void abort(String why, Throwable e) {
3675 aborted = true;
3676 throw new RuntimeException("Fatal ZK error, why=" + why, e);
3677 }
3678
3679 @Override
3680 public boolean isAborted() {
3681 return aborted;
3682 }
3683 });
3684 return zkw;
3685 }
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698 public static ZooKeeperWatcher createAndForceNodeToOpenedState(
3699 HBaseTestingUtility TEST_UTIL, Region region,
3700 ServerName serverName) throws ZooKeeperConnectionException,
3701 IOException, KeeperException, NodeExistsException {
3702 return createAndForceNodeToOpenedState(TEST_UTIL, (HRegion)region, serverName);
3703 }
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716 public static ZooKeeperWatcher createAndForceNodeToOpenedState(
3717 HBaseTestingUtility TEST_UTIL, HRegion region,
3718 ServerName serverName) throws ZooKeeperConnectionException,
3719 IOException, KeeperException, NodeExistsException {
3720 ZooKeeperWatcher zkw = getZooKeeperWatcher(TEST_UTIL);
3721 ZKAssign.createNodeOffline(zkw, region.getRegionInfo(), serverName);
3722 int version = ZKAssign.transitionNodeOpening(zkw, region
3723 .getRegionInfo(), serverName);
3724 ZKAssign.transitionNodeOpened(zkw, region.getRegionInfo(), serverName,
3725 version);
3726 return zkw;
3727 }
3728
3729 public static void assertKVListsEqual(String additionalMsg,
3730 final List<? extends Cell> expected,
3731 final List<? extends Cell> actual) {
3732 final int eLen = expected.size();
3733 final int aLen = actual.size();
3734 final int minLen = Math.min(eLen, aLen);
3735
3736 int i;
3737 for (i = 0; i < minLen
3738 && KeyValue.COMPARATOR.compare(expected.get(i), actual.get(i)) == 0;
3739 ++i) {}
3740
3741 if (additionalMsg == null) {
3742 additionalMsg = "";
3743 }
3744 if (!additionalMsg.isEmpty()) {
3745 additionalMsg = ". " + additionalMsg;
3746 }
3747
3748 if (eLen != aLen || i != minLen) {
3749 throw new AssertionError(
3750 "Expected and actual KV arrays differ at position " + i + ": " +
3751 safeGetAsStr(expected, i) + " (length " + eLen +") vs. " +
3752 safeGetAsStr(actual, i) + " (length " + aLen + ")" + additionalMsg);
3753 }
3754 }
3755
3756 public static <T> String safeGetAsStr(List<T> lst, int i) {
3757 if (0 <= i && i < lst.size()) {
3758 return lst.get(i).toString();
3759 } else {
3760 return "<out_of_range>";
3761 }
3762 }
3763
3764 public String getClusterKey() {
3765 return conf.get(HConstants.ZOOKEEPER_QUORUM) + ":"
3766 + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT) + ":"
3767 + conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
3768 HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
3769 }
3770
3771
3772 public HTable createRandomTable(String tableName,
3773 final Collection<String> families,
3774 final int maxVersions,
3775 final int numColsPerRow,
3776 final int numFlushes,
3777 final int numRegions,
3778 final int numRowsPerFlush)
3779 throws IOException, InterruptedException {
3780
3781 LOG.info("\n\nCreating random table " + tableName + " with " + numRegions +
3782 " regions, " + numFlushes + " storefiles per region, " +
3783 numRowsPerFlush + " rows per flush, maxVersions=" + maxVersions +
3784 "\n");
3785
3786 final Random rand = new Random(tableName.hashCode() * 17L + 12938197137L);
3787 final int numCF = families.size();
3788 final byte[][] cfBytes = new byte[numCF][];
3789 {
3790 int cfIndex = 0;
3791 for (String cf : families) {
3792 cfBytes[cfIndex++] = Bytes.toBytes(cf);
3793 }
3794 }
3795
3796 final int actualStartKey = 0;
3797 final int actualEndKey = Integer.MAX_VALUE;
3798 final int keysPerRegion = (actualEndKey - actualStartKey) / numRegions;
3799 final int splitStartKey = actualStartKey + keysPerRegion;
3800 final int splitEndKey = actualEndKey - keysPerRegion;
3801 final String keyFormat = "%08x";
3802 final HTable table = createTable(tableName, cfBytes,
3803 maxVersions,
3804 Bytes.toBytes(String.format(keyFormat, splitStartKey)),
3805 Bytes.toBytes(String.format(keyFormat, splitEndKey)),
3806 numRegions);
3807
3808 if (hbaseCluster != null) {
3809 getMiniHBaseCluster().flushcache(TableName.META_TABLE_NAME);
3810 }
3811
3812 for (int iFlush = 0; iFlush < numFlushes; ++iFlush) {
3813 for (int iRow = 0; iRow < numRowsPerFlush; ++iRow) {
3814 final byte[] row = Bytes.toBytes(String.format(keyFormat,
3815 actualStartKey + rand.nextInt(actualEndKey - actualStartKey)));
3816
3817 Put put = new Put(row);
3818 Delete del = new Delete(row);
3819 for (int iCol = 0; iCol < numColsPerRow; ++iCol) {
3820 final byte[] cf = cfBytes[rand.nextInt(numCF)];
3821 final long ts = rand.nextInt();
3822 final byte[] qual = Bytes.toBytes("col" + iCol);
3823 if (rand.nextBoolean()) {
3824 final byte[] value = Bytes.toBytes("value_for_row_" + iRow +
3825 "_cf_" + Bytes.toStringBinary(cf) + "_col_" + iCol + "_ts_" +
3826 ts + "_random_" + rand.nextLong());
3827 put.add(cf, qual, ts, value);
3828 } else if (rand.nextDouble() < 0.8) {
3829 del.deleteColumn(cf, qual, ts);
3830 } else {
3831 del.deleteColumns(cf, qual, ts);
3832 }
3833 }
3834
3835 if (!put.isEmpty()) {
3836 table.put(put);
3837 }
3838
3839 if (!del.isEmpty()) {
3840 table.delete(del);
3841 }
3842 }
3843 LOG.info("Initiating flush #" + iFlush + " for table " + tableName);
3844 table.flushCommits();
3845 if (hbaseCluster != null) {
3846 getMiniHBaseCluster().flushcache(table.getName());
3847 }
3848 }
3849
3850 return table;
3851 }
3852
3853 private static final int MIN_RANDOM_PORT = 0xc000;
3854 private static final int MAX_RANDOM_PORT = 0xfffe;
3855 private static Random random = new Random();
3856
3857
3858
3859
3860
3861 public static int randomPort() {
3862 return MIN_RANDOM_PORT
3863 + random.nextInt(MAX_RANDOM_PORT - MIN_RANDOM_PORT);
3864 }
3865
3866
3867
3868
3869
3870 public static int randomFreePort() {
3871 int port = 0;
3872 do {
3873 port = randomPort();
3874 if (takenRandomPorts.contains(port)) {
3875 port = 0;
3876 continue;
3877 }
3878 takenRandomPorts.add(port);
3879
3880 try {
3881 ServerSocket sock = new ServerSocket(port);
3882 sock.close();
3883 } catch (IOException ex) {
3884 port = 0;
3885 }
3886 } while (port == 0);
3887 return port;
3888 }
3889
3890
3891 public static String randomMultiCastAddress() {
3892 return "226.1.1." + random.nextInt(254);
3893 }
3894
3895
3896
3897 public static void waitForHostPort(String host, int port)
3898 throws IOException {
3899 final int maxTimeMs = 10000;
3900 final int maxNumAttempts = maxTimeMs / HConstants.SOCKET_RETRY_WAIT_MS;
3901 IOException savedException = null;
3902 LOG.info("Waiting for server at " + host + ":" + port);
3903 for (int attempt = 0; attempt < maxNumAttempts; ++attempt) {
3904 try {
3905 Socket sock = new Socket(InetAddress.getByName(host), port);
3906 sock.close();
3907 savedException = null;
3908 LOG.info("Server at " + host + ":" + port + " is available");
3909 break;
3910 } catch (UnknownHostException e) {
3911 throw new IOException("Failed to look up " + host, e);
3912 } catch (IOException e) {
3913 savedException = e;
3914 }
3915 Threads.sleepWithoutInterrupt(HConstants.SOCKET_RETRY_WAIT_MS);
3916 }
3917
3918 if (savedException != null) {
3919 throw savedException;
3920 }
3921 }
3922
3923
3924
3925
3926
3927
3928 public static int createPreSplitLoadTestTable(Configuration conf,
3929 TableName tableName, byte[] columnFamily, Algorithm compression,
3930 DataBlockEncoding dataBlockEncoding) throws IOException {
3931 return createPreSplitLoadTestTable(conf, tableName,
3932 columnFamily, compression, dataBlockEncoding, DEFAULT_REGIONS_PER_SERVER, 1,
3933 Durability.USE_DEFAULT);
3934 }
3935
3936
3937
3938
3939
3940 public static int createPreSplitLoadTestTable(Configuration conf,
3941 TableName tableName, byte[] columnFamily, Algorithm compression,
3942 DataBlockEncoding dataBlockEncoding, int numRegionsPerServer, int regionReplication,
3943 Durability durability)
3944 throws IOException {
3945 HTableDescriptor desc = new HTableDescriptor(tableName);
3946 desc.setDurability(durability);
3947 desc.setRegionReplication(regionReplication);
3948 HColumnDescriptor hcd = new HColumnDescriptor(columnFamily);
3949 hcd.setDataBlockEncoding(dataBlockEncoding);
3950 hcd.setCompressionType(compression);
3951 return createPreSplitLoadTestTable(conf, desc, hcd, numRegionsPerServer);
3952 }
3953
3954
3955
3956
3957
3958
3959 public static int createPreSplitLoadTestTable(Configuration conf,
3960 TableName tableName, byte[][] columnFamilies, Algorithm compression,
3961 DataBlockEncoding dataBlockEncoding, int numRegionsPerServer, int regionReplication,
3962 Durability durability)
3963 throws IOException {
3964 HTableDescriptor desc = new HTableDescriptor(tableName);
3965 desc.setDurability(durability);
3966 desc.setRegionReplication(regionReplication);
3967 HColumnDescriptor[] hcds = new HColumnDescriptor[columnFamilies.length];
3968 for (int i = 0; i < columnFamilies.length; i++) {
3969 HColumnDescriptor hcd = new HColumnDescriptor(columnFamilies[i]);
3970 hcd.setDataBlockEncoding(dataBlockEncoding);
3971 hcd.setCompressionType(compression);
3972 hcds[i] = hcd;
3973 }
3974 return createPreSplitLoadTestTable(conf, desc, hcds, numRegionsPerServer);
3975 }
3976
3977
3978
3979
3980
3981
3982 public static int createPreSplitLoadTestTable(Configuration conf,
3983 HTableDescriptor desc, HColumnDescriptor hcd) throws IOException {
3984 return createPreSplitLoadTestTable(conf, desc, hcd, DEFAULT_REGIONS_PER_SERVER);
3985 }
3986
3987
3988
3989
3990
3991
3992 public static int createPreSplitLoadTestTable(Configuration conf,
3993 HTableDescriptor desc, HColumnDescriptor hcd, int numRegionsPerServer) throws IOException {
3994 return createPreSplitLoadTestTable(conf, desc, new HColumnDescriptor[] {hcd},
3995 numRegionsPerServer);
3996 }
3997
3998
3999
4000
4001
4002
4003 public static int createPreSplitLoadTestTable(Configuration conf,
4004 HTableDescriptor desc, HColumnDescriptor[] hcds, int numRegionsPerServer) throws IOException {
4005 for (HColumnDescriptor hcd : hcds) {
4006 if (!desc.hasFamily(hcd.getName())) {
4007 desc.addFamily(hcd);
4008 }
4009 }
4010
4011 int totalNumberOfRegions = 0;
4012 Connection unmanagedConnection = ConnectionFactory.createConnection(conf);
4013 Admin admin = unmanagedConnection.getAdmin();
4014
4015 try {
4016
4017
4018
4019 int numberOfServers = admin.getClusterStatus().getServers().size();
4020 if (numberOfServers == 0) {
4021 throw new IllegalStateException("No live regionservers");
4022 }
4023
4024 totalNumberOfRegions = numberOfServers * numRegionsPerServer;
4025 LOG.info("Number of live regionservers: " + numberOfServers + ", " +
4026 "pre-splitting table into " + totalNumberOfRegions + " regions " +
4027 "(regions per server: " + numRegionsPerServer + ")");
4028
4029 byte[][] splits = new RegionSplitter.HexStringSplit().split(
4030 totalNumberOfRegions);
4031
4032 admin.createTable(desc, splits);
4033 } catch (MasterNotRunningException e) {
4034 LOG.error("Master not running", e);
4035 throw new IOException(e);
4036 } catch (TableExistsException e) {
4037 LOG.warn("Table " + desc.getTableName() +
4038 " already exists, continuing");
4039 } finally {
4040 admin.close();
4041 unmanagedConnection.close();
4042 }
4043 return totalNumberOfRegions;
4044 }
4045
4046 public static int getMetaRSPort(Configuration conf) throws IOException {
4047 try (Connection c = ConnectionFactory.createConnection();
4048 RegionLocator locator = c.getRegionLocator(TableName.META_TABLE_NAME)) {
4049 return locator.getRegionLocation(Bytes.toBytes("")).getPort();
4050 }
4051 }
4052
4053
4054
4055
4056
4057
4058
4059 public void assertRegionOnServer(
4060 final HRegionInfo hri, final ServerName server,
4061 final long timeout) throws IOException, InterruptedException {
4062 long timeoutTime = System.currentTimeMillis() + timeout;
4063 while (true) {
4064 List<HRegionInfo> regions = getHBaseAdmin().getOnlineRegions(server);
4065 if (regions.contains(hri)) return;
4066 long now = System.currentTimeMillis();
4067 if (now > timeoutTime) break;
4068 Thread.sleep(10);
4069 }
4070 fail("Could not find region " + hri.getRegionNameAsString()
4071 + " on server " + server);
4072 }
4073
4074
4075
4076
4077
4078 public void assertRegionOnlyOnServer(
4079 final HRegionInfo hri, final ServerName server,
4080 final long timeout) throws IOException, InterruptedException {
4081 long timeoutTime = System.currentTimeMillis() + timeout;
4082 while (true) {
4083 List<HRegionInfo> regions = getHBaseAdmin().getOnlineRegions(server);
4084 if (regions.contains(hri)) {
4085 List<JVMClusterUtil.RegionServerThread> rsThreads =
4086 getHBaseCluster().getLiveRegionServerThreads();
4087 for (JVMClusterUtil.RegionServerThread rsThread: rsThreads) {
4088 HRegionServer rs = rsThread.getRegionServer();
4089 if (server.equals(rs.getServerName())) {
4090 continue;
4091 }
4092 Collection<Region> hrs = rs.getOnlineRegionsLocalContext();
4093 for (Region r: hrs) {
4094 assertTrue("Region should not be double assigned",
4095 r.getRegionInfo().getRegionId() != hri.getRegionId());
4096 }
4097 }
4098 return;
4099 }
4100 long now = System.currentTimeMillis();
4101 if (now > timeoutTime) break;
4102 Thread.sleep(10);
4103 }
4104 fail("Could not find region " + hri.getRegionNameAsString()
4105 + " on server " + server);
4106 }
4107
4108 public HRegion createTestRegion(String tableName, HColumnDescriptor hcd)
4109 throws IOException {
4110 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
4111 htd.addFamily(hcd);
4112 HRegionInfo info =
4113 new HRegionInfo(TableName.valueOf(tableName), null, null, false);
4114 HRegion region =
4115 HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), htd);
4116 return region;
4117 }
4118
4119 public void setFileSystemURI(String fsURI) {
4120 FS_URI = fsURI;
4121 }
4122
4123
4124
4125
4126 public <E extends Exception> long waitFor(long timeout, Predicate<E> predicate)
4127 throws E {
4128 return Waiter.waitFor(this.conf, timeout, predicate);
4129 }
4130
4131
4132
4133
4134 public <E extends Exception> long waitFor(long timeout, long interval, Predicate<E> predicate)
4135 throws E {
4136 return Waiter.waitFor(this.conf, timeout, interval, predicate);
4137 }
4138
4139
4140
4141
4142 public <E extends Exception> long waitFor(long timeout, long interval,
4143 boolean failIfTimeout, Predicate<E> predicate) throws E {
4144 return Waiter.waitFor(this.conf, timeout, interval, failIfTimeout, predicate);
4145 }
4146
4147
4148
4149
4150 public ExplainingPredicate<IOException> predicateNoRegionsInTransition() {
4151 return new ExplainingPredicate<IOException>() {
4152 @Override
4153 public String explainFailure() throws IOException {
4154 final RegionStates regionStates = getMiniHBaseCluster().getMaster()
4155 .getAssignmentManager().getRegionStates();
4156 return "found in transition: " + regionStates.getRegionsInTransition().toString();
4157 }
4158
4159 @Override
4160 public boolean evaluate() throws IOException {
4161 HMaster master = getMiniHBaseCluster().getMaster();
4162 if (master == null) return false;
4163 AssignmentManager am = master.getAssignmentManager();
4164 if (am == null) return false;
4165 final RegionStates regionStates = am.getRegionStates();
4166 return !regionStates.isRegionsInTransition();
4167 }
4168 };
4169 }
4170
4171
4172
4173
4174 public Waiter.Predicate<IOException> predicateTableEnabled(final TableName tableName) {
4175 return new ExplainingPredicate<IOException>() {
4176 @Override
4177 public String explainFailure() throws IOException {
4178 return explainTableState(tableName);
4179 }
4180
4181 @Override
4182 public boolean evaluate() throws IOException {
4183 return getHBaseAdmin().tableExists(tableName) && getHBaseAdmin().isTableEnabled(tableName);
4184 }
4185 };
4186 }
4187
4188
4189
4190
4191 public Waiter.Predicate<IOException> predicateTableDisabled(final TableName tableName) {
4192 return new ExplainingPredicate<IOException>() {
4193 @Override
4194 public String explainFailure() throws IOException {
4195 return explainTableState(tableName);
4196 }
4197
4198 @Override
4199 public boolean evaluate() throws IOException {
4200 return getHBaseAdmin().isTableDisabled(tableName);
4201 }
4202 };
4203 }
4204
4205
4206
4207
4208 public Waiter.Predicate<IOException> predicateTableAvailable(final TableName tableName) {
4209 return new ExplainingPredicate<IOException>() {
4210 @Override
4211 public String explainFailure() throws IOException {
4212 return explainTableAvailability(tableName);
4213 }
4214
4215 @Override
4216 public boolean evaluate() throws IOException {
4217 boolean tableAvailable = getHBaseAdmin().isTableAvailable(tableName);
4218 if (tableAvailable) {
4219 try (Table table = getConnection().getTable(tableName)) {
4220 HTableDescriptor htd = table.getTableDescriptor();
4221 for (HRegionLocation loc : getConnection().getRegionLocator(tableName)
4222 .getAllRegionLocations()) {
4223 Scan scan = new Scan().withStartRow(loc.getRegionInfo().getStartKey())
4224 .withStopRow(loc.getRegionInfo().getEndKey()).setOneRowLimit()
4225 .setMaxResultsPerColumnFamily(1).setCacheBlocks(false);
4226 for (byte[] family : htd.getFamiliesKeys()) {
4227 scan.addFamily(family);
4228 }
4229 try (ResultScanner scanner = table.getScanner(scan)) {
4230 scanner.next();
4231 }
4232 }
4233 }
4234 }
4235 return tableAvailable;
4236 }
4237 };
4238 }
4239
4240
4241
4242
4243
4244
4245 public void waitUntilNoRegionsInTransition(final long timeout) throws IOException {
4246 waitFor(timeout, predicateNoRegionsInTransition());
4247 }
4248
4249
4250
4251
4252
4253 public void waitUntilNoRegionsInTransition() throws IOException {
4254 waitUntilNoRegionsInTransition(15 * 60000);
4255 }
4256
4257
4258
4259
4260
4261
4262 public void waitLabelAvailable(long timeoutMillis, final String... labels) {
4263 final VisibilityLabelsCache labelsCache = VisibilityLabelsCache.get();
4264 waitFor(timeoutMillis, new Waiter.ExplainingPredicate<RuntimeException>() {
4265
4266 @Override
4267 public boolean evaluate() {
4268 for (String label : labels) {
4269 if (labelsCache.getLabelOrdinal(label) == 0) {
4270 return false;
4271 }
4272 }
4273 return true;
4274 }
4275
4276 @Override
4277 public String explainFailure() {
4278 for (String label : labels) {
4279 if (labelsCache.getLabelOrdinal(label) == 0) {
4280 return label + " is not available yet";
4281 }
4282 }
4283 return "";
4284 }
4285 });
4286 }
4287
4288
4289
4290
4291
4292
4293 public static List<HColumnDescriptor> generateColumnDescriptors() {
4294 return generateColumnDescriptors("");
4295 }
4296
4297
4298
4299
4300
4301
4302
4303 public static List<HColumnDescriptor> generateColumnDescriptors(final String prefix) {
4304 List<HColumnDescriptor> htds = new ArrayList<HColumnDescriptor>();
4305 long familyId = 0;
4306 for (Compression.Algorithm compressionType: getSupportedCompressionAlgorithms()) {
4307 for (DataBlockEncoding encodingType: DataBlockEncoding.values()) {
4308 for (BloomType bloomType: BloomType.values()) {
4309 String name = String.format("%s-cf-!@#&-%d!@#", prefix, familyId);
4310 HColumnDescriptor htd = new HColumnDescriptor(name);
4311 htd.setCompressionType(compressionType);
4312 htd.setDataBlockEncoding(encodingType);
4313 htd.setBloomFilterType(bloomType);
4314 htds.add(htd);
4315 familyId++;
4316 }
4317 }
4318 }
4319 return htds;
4320 }
4321
4322
4323
4324
4325
4326 public static Compression.Algorithm[] getSupportedCompressionAlgorithms() {
4327 String[] allAlgos = HFile.getSupportedCompressionAlgorithms();
4328 List<Compression.Algorithm> supportedAlgos = new ArrayList<Compression.Algorithm>();
4329 for (String algoName : allAlgos) {
4330 try {
4331 Compression.Algorithm algo = Compression.getCompressionAlgorithmByName(algoName);
4332 algo.getCompressor();
4333 supportedAlgos.add(algo);
4334 } catch (Throwable t) {
4335
4336 }
4337 }
4338 return supportedAlgos.toArray(new Algorithm[supportedAlgos.size()]);
4339 }
4340
4341
4342
4343
4344
4345
4346
4347 public MiniKdc setupMiniKdc(File keytabFile) throws Exception {
4348 Properties conf = MiniKdc.createConf();
4349 conf.put(MiniKdc.DEBUG, true);
4350 MiniKdc kdc = null;
4351 File dir = null;
4352
4353
4354 boolean bindException;
4355 int numTries = 0;
4356 do {
4357 try {
4358 bindException = false;
4359 dir = new File(getDataTestDir("kdc").toUri().getPath());
4360 kdc = new MiniKdc(conf, dir);
4361 kdc.start();
4362 } catch (BindException e) {
4363 FileUtils.deleteDirectory(dir);
4364 numTries++;
4365 if (numTries == 3) {
4366 LOG.error("Failed setting up MiniKDC. Tried " + numTries + " times.");
4367 throw e;
4368 }
4369 LOG.error("BindException encountered when setting up MiniKdc. Trying again.");
4370 bindException = true;
4371 }
4372 } while (bindException);
4373 HBaseKerberosUtils.setKeytabFileForTesting(keytabFile.getAbsolutePath());
4374 return kdc;
4375 }
4376
4377 public int getNumHFiles(final TableName tableName, final byte[] family) {
4378 int numHFiles = 0;
4379 for (RegionServerThread regionServerThread : getMiniHBaseCluster().getRegionServerThreads()) {
4380 numHFiles+= getNumHFilesForRS(regionServerThread.getRegionServer(), tableName,
4381 family);
4382 }
4383 return numHFiles;
4384 }
4385
4386 public int getNumHFilesForRS(final HRegionServer rs, final TableName tableName,
4387 final byte[] family) {
4388 int numHFiles = 0;
4389 for (Region region : rs.getOnlineRegions(tableName)) {
4390 numHFiles += region.getStore(family).getStorefilesCount();
4391 }
4392 return numHFiles;
4393 }
4394 }