1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertFalse;
24 import static org.junit.Assert.assertTrue;
25
26 import java.io.File;
27 import java.io.IOException;
28 import java.util.HashMap;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.Map.Entry;
32 import java.util.Random;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.FileUtil;
38 import org.apache.hadoop.fs.Path;
39 import org.apache.hadoop.hbase.client.Get;
40 import org.apache.hadoop.hbase.client.Put;
41 import org.apache.hadoop.hbase.client.Result;
42 import org.apache.hadoop.hbase.client.Table;
43 import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
44 import org.apache.hadoop.hbase.testclassification.LargeTests;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
47 import org.apache.hadoop.hdfs.MiniDFSCluster;
48 import org.junit.Test;
49 import org.junit.experimental.categories.Category;
50
51
52
53
54 @Category(LargeTests.class)
55 public class TestHBaseTestingUtility {
56 private static final Log LOG = LogFactory.getLog(TestHBaseTestingUtility.class);
57
58
59
60
61
62
63
64 @Test (timeout=180000)
65 public void testMultiClusters() throws Exception {
66
67
68
69 HBaseTestingUtility htu1 = new HBaseTestingUtility();
70
71 htu1.getConfiguration().set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
72 htu1.startMiniZKCluster();
73
74
75 HBaseTestingUtility htu2 = new HBaseTestingUtility();
76 htu2.getConfiguration().set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
77 htu2.getConfiguration().set(HConstants.ZOOKEEPER_CLIENT_PORT,
78 htu1.getConfiguration().get(HConstants.ZOOKEEPER_CLIENT_PORT, "-1"));
79 htu2.setZkCluster(htu1.getZkCluster());
80
81
82
83
84 HBaseTestingUtility htu3 = new HBaseTestingUtility();
85 htu3.getConfiguration().set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/3");
86 htu3.getConfiguration().set(HConstants.ZOOKEEPER_CLIENT_PORT,
87 htu1.getConfiguration().get(HConstants.ZOOKEEPER_CLIENT_PORT, "-1"));
88 htu3.setZkCluster(htu1.getZkCluster());
89
90 try {
91 htu1.startMiniCluster();
92 htu2.startMiniCluster();
93 htu3.startMiniCluster();
94
95 final TableName TABLE_NAME = TableName.valueOf("test");
96 final byte[] FAM_NAME = Bytes.toBytes("fam");
97 final byte[] ROW = Bytes.toBytes("row");
98 final byte[] QUAL_NAME = Bytes.toBytes("qual");
99 final byte[] VALUE = Bytes.toBytes("value");
100
101 Table table1 = htu1.createTable(TABLE_NAME, FAM_NAME);
102 Table table2 = htu2.createTable(TABLE_NAME, FAM_NAME);
103
104 Put put = new Put(ROW);
105 put.add(FAM_NAME, QUAL_NAME, VALUE);
106 table1.put(put);
107
108 Get get = new Get(ROW);
109 get.addColumn(FAM_NAME, QUAL_NAME);
110 Result res = table1.get(get);
111 assertEquals(1, res.size());
112
113 res = table2.get(get);
114 assertEquals(0, res.size());
115
116 table1.close();
117 table2.close();
118
119 } finally {
120 htu3.shutdownMiniCluster();
121 htu2.shutdownMiniCluster();
122 htu1.shutdownMiniCluster();
123 }
124 }
125
126 @Test public void testMiniCluster() throws Exception {
127 HBaseTestingUtility hbt = new HBaseTestingUtility();
128
129 MiniHBaseCluster cluster = hbt.startMiniCluster();
130 try {
131 assertEquals(1, cluster.getLiveRegionServerThreads().size());
132 } finally {
133 hbt.shutdownMiniCluster();
134 }
135 }
136
137 @Test
138 public void testMiniClusterBindToWildcard() throws Exception {
139 HBaseTestingUtility hbt = new HBaseTestingUtility();
140 hbt.getConfiguration().set("hbase.regionserver.ipc.address", "0.0.0.0");
141 MiniHBaseCluster cluster = hbt.startMiniCluster();
142 try {
143 assertEquals(1, cluster.getLiveRegionServerThreads().size());
144 } finally {
145 hbt.shutdownMiniCluster();
146 }
147 }
148
149 @Test
150 public void testMiniClusterWithSSLOn() throws Exception {
151 final String BASEDIR = System.getProperty("test.build.dir",
152 "target/test-dir") + "/" + TestHBaseTestingUtility.class.getSimpleName();
153 String sslConfDir = KeyStoreTestUtil.getClasspathDir(TestHBaseTestingUtility.class);
154 String keystoresDir = new File(BASEDIR).getAbsolutePath();
155
156 HBaseTestingUtility hbt = new HBaseTestingUtility();
157 File base = new File(BASEDIR);
158 FileUtil.fullyDelete(base);
159 base.mkdirs();
160
161 KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, hbt.getConfiguration(), false);
162
163 hbt.getConfiguration().set("hbase.ssl.enabled", "true");
164 hbt.getConfiguration().addResource("ssl-server.xml");
165 hbt.getConfiguration().addResource("ssl-client.xml");
166
167 MiniHBaseCluster cluster = hbt.startMiniCluster();
168 try {
169 assertEquals(1, cluster.getLiveRegionServerThreads().size());
170 } finally {
171 hbt.shutdownMiniCluster();
172 }
173 }
174
175
176
177
178
179 @Test public void testMultipleStartStop() throws Exception{
180 HBaseTestingUtility htu1 = new HBaseTestingUtility();
181 Path foo = new Path("foo");
182
183 htu1.startMiniCluster();
184 htu1.getDFSCluster().getFileSystem().create(foo);
185 assertTrue( htu1.getDFSCluster().getFileSystem().exists(foo));
186 htu1.shutdownMiniCluster();
187
188 htu1.startMiniCluster();
189 assertFalse( htu1.getDFSCluster().getFileSystem().exists(foo));
190 htu1.getDFSCluster().getFileSystem().create(foo);
191 assertTrue( htu1.getDFSCluster().getFileSystem().exists(foo));
192 htu1.shutdownMiniCluster();
193 }
194
195 @Test
196 public void testMiniZooKeeperWithOneServer() throws Exception {
197 HBaseTestingUtility hbt = new HBaseTestingUtility();
198 MiniZooKeeperCluster cluster1 = hbt.startMiniZKCluster();
199 try {
200 assertEquals(0, cluster1.getBackupZooKeeperServerNum());
201 assertTrue((cluster1.killCurrentActiveZooKeeperServer() == -1));
202 } finally {
203 hbt.shutdownMiniZKCluster();
204 }
205 }
206
207 @Test
208 public void testMiniZooKeeperWithMultipleServers() throws Exception {
209 HBaseTestingUtility hbt = new HBaseTestingUtility();
210
211 MiniZooKeeperCluster cluster2 = hbt.startMiniZKCluster(5);
212 int defaultClientPort = 21818;
213 cluster2.setDefaultClientPort(defaultClientPort);
214 try {
215 assertEquals(4, cluster2.getBackupZooKeeperServerNum());
216
217
218 int currentActivePort = cluster2.killCurrentActiveZooKeeperServer();
219 assertTrue(currentActivePort >= defaultClientPort);
220
221 assertTrue(cluster2.getClientPort() == currentActivePort);
222
223
224 currentActivePort = cluster2.killCurrentActiveZooKeeperServer();
225 assertTrue(currentActivePort >= defaultClientPort);
226 assertTrue(cluster2.getClientPort() == currentActivePort);
227 assertEquals(2, cluster2.getBackupZooKeeperServerNum());
228 assertEquals(3, cluster2.getZooKeeperServerNum());
229
230
231 cluster2.killOneBackupZooKeeperServer();
232 cluster2.killOneBackupZooKeeperServer();
233 assertEquals(0, cluster2.getBackupZooKeeperServerNum());
234 assertEquals(1, cluster2.getZooKeeperServerNum());
235
236
237 currentActivePort = cluster2.killCurrentActiveZooKeeperServer();
238 assertTrue(currentActivePort == -1);
239 assertTrue(cluster2.getClientPort() == currentActivePort);
240
241 cluster2.killOneBackupZooKeeperServer();
242 assertEquals(-1, cluster2.getBackupZooKeeperServerNum());
243 assertEquals(0, cluster2.getZooKeeperServerNum());
244 } finally {
245 hbt.shutdownMiniZKCluster();
246 }
247 }
248
249 @Test
250 public void testMiniZooKeeperWithMultipleClientPorts() throws Exception {
251 int defaultClientPort = 8888;
252 int i, j;
253 HBaseTestingUtility hbt = new HBaseTestingUtility();
254
255
256 int [] clientPortList1 = {1111, 1112, 1113};
257 MiniZooKeeperCluster cluster1 = hbt.startMiniZKCluster(clientPortList1.length, clientPortList1);
258 try {
259 List<Integer> clientPortListInCluster = cluster1.getClientPortList();
260
261 for (i = 0; i < clientPortListInCluster.size(); i++) {
262 assertEquals(clientPortListInCluster.get(i).intValue(), clientPortList1[i]);
263 }
264 } finally {
265 hbt.shutdownMiniZKCluster();
266 }
267
268
269 hbt.getConfiguration().setInt("test.hbase.zookeeper.property.clientPort", defaultClientPort);
270 int [] clientPortList2 = {2222, 2223};
271 MiniZooKeeperCluster cluster2 =
272 hbt.startMiniZKCluster(clientPortList2.length + 2, clientPortList2);
273
274 try {
275 List<Integer> clientPortListInCluster = cluster2.getClientPortList();
276
277 for (i = 0, j = 0; i < clientPortListInCluster.size(); i++) {
278 if (i < clientPortList2.length) {
279 assertEquals(clientPortListInCluster.get(i).intValue(), clientPortList2[i]);
280 } else {
281
282
283 assertEquals(clientPortListInCluster.get(i).intValue(), defaultClientPort + j);
284 j++;
285 }
286 }
287 } finally {
288 hbt.shutdownMiniZKCluster();
289 }
290
291
292 hbt.getConfiguration().setInt("test.hbase.zookeeper.property.clientPort", defaultClientPort);
293 int [] clientPortList3 = {3333, -3334, 3335, 0};
294 MiniZooKeeperCluster cluster3 =
295 hbt.startMiniZKCluster(clientPortList3.length + 1, clientPortList3);
296
297 try {
298 List<Integer> clientPortListInCluster = cluster3.getClientPortList();
299
300 for (i = 0, j = 0; i < clientPortListInCluster.size(); i++) {
301
302
303 if (i < clientPortList3.length && clientPortList3[i] > 0) {
304 assertEquals(clientPortListInCluster.get(i).intValue(), clientPortList3[i]);
305 } else {
306 assertEquals(clientPortListInCluster.get(i).intValue(), defaultClientPort + j);
307 j++;
308 }
309 }
310 } finally {
311 hbt.shutdownMiniZKCluster();
312 }
313
314
315
316
317
318 hbt.getConfiguration().setInt("test.hbase.zookeeper.property.clientPort", defaultClientPort);
319 int [] clientPortList4 = {-4444, defaultClientPort+2, 4446, defaultClientPort};
320 MiniZooKeeperCluster cluster4 =
321 hbt.startMiniZKCluster(clientPortList4.length + 1, clientPortList4);
322
323 try {
324 List<Integer> clientPortListInCluster = cluster4.getClientPortList();
325
326 for (i = 0, j = 1; i < clientPortListInCluster.size(); i++) {
327
328
329 if (i < clientPortList4.length && clientPortList4[i] > 0) {
330 assertEquals(clientPortListInCluster.get(i).intValue(), clientPortList4[i]);
331 } else {
332 assertEquals(clientPortListInCluster.get(i).intValue(), defaultClientPort + j);
333 j +=2;
334 }
335 }
336 } finally {
337 hbt.shutdownMiniZKCluster();
338 }
339
340
341 int [] clientPortList5 = {5555, 5556, 5556};
342
343 try {
344 MiniZooKeeperCluster cluster5 =
345 hbt.startMiniZKCluster(clientPortList5.length, clientPortList5);
346 assertTrue(cluster5.getClientPort() == -1);
347 } catch (Exception e) {
348
349 } finally {
350 hbt.shutdownMiniZKCluster();
351 }
352 }
353
354 @Test public void testMiniDFSCluster() throws Exception {
355 HBaseTestingUtility hbt = new HBaseTestingUtility();
356 MiniDFSCluster cluster = hbt.startMiniDFSCluster(null);
357 FileSystem dfs = cluster.getFileSystem();
358 Path dir = new Path("dir");
359 Path qualifiedDir = dfs.makeQualified(dir);
360 LOG.info("dir=" + dir + ", qualifiedDir=" + qualifiedDir);
361 assertFalse(dfs.exists(qualifiedDir));
362 assertTrue(dfs.mkdirs(qualifiedDir));
363 assertTrue(dfs.delete(qualifiedDir, true));
364 hbt.shutdownMiniCluster();
365 }
366
367 @Test public void testSetupClusterTestBuildDir() throws Exception {
368 HBaseTestingUtility hbt = new HBaseTestingUtility();
369 Path testdir = hbt.getClusterTestDir();
370 LOG.info("uuid-subdir=" + testdir);
371 FileSystem fs = hbt.getTestFileSystem();
372
373 assertFalse(fs.exists(testdir));
374
375 hbt.startMiniDFSCluster(null);
376 assertTrue(fs.exists(testdir));
377
378 hbt.shutdownMiniCluster();
379 assertFalse(fs.exists(testdir));
380 }
381
382 @Test public void testTestDir() throws Exception {
383 HBaseTestingUtility hbt = new HBaseTestingUtility();
384 Path testdir = hbt.getDataTestDir();
385 LOG.info("testdir=" + testdir);
386 FileSystem fs = hbt.getTestFileSystem();
387 assertTrue(!fs.exists(testdir));
388 assertTrue(fs.mkdirs(testdir));
389 assertTrue(hbt.cleanupTestDir());
390 }
391
392 @Test public void testMRYarnConfigsPopulation() throws IOException {
393 Map<String, String> dummyProps = new HashMap<>();
394 dummyProps.put("mapreduce.jobtracker.address", "dummyhost:11234");
395 dummyProps.put("yarn.resourcemanager.address", "dummyhost:11235");
396 dummyProps.put("mapreduce.jobhistory.address", "dummyhost:11236");
397 dummyProps.put("yarn.resourcemanager.scheduler.address", "dummyhost:11237");
398 dummyProps.put("mapreduce.jobhistory.webapp.address", "dummyhost:11238");
399 dummyProps.put("yarn.resourcemanager.webapp.address", "dummyhost:11239");
400
401 HBaseTestingUtility hbt = new HBaseTestingUtility();
402
403
404 for (Entry<String, String> entry : dummyProps.entrySet()) {
405 hbt.getConfiguration().set(entry.getKey(), entry.getValue());
406 }
407
408 for (Entry<String,String> entry : dummyProps.entrySet()) {
409 assertTrue("The Configuration for key " + entry.getKey() +" and value: " + entry.getValue() +
410 " is not populated correctly", hbt.getConfiguration().get(entry.getKey()).equals(entry.getValue()));
411 }
412
413 hbt.startMiniMapReduceCluster();
414
415
416 for (Entry<String,String> entry : dummyProps.entrySet()) {
417 assertFalse("The MR prop: " + entry.getValue() + " is not overwritten when map reduce mini"+
418 "cluster is started", hbt.getConfiguration().get(entry.getKey()).equals(entry.getValue()));
419 }
420
421 hbt.shutdownMiniMapReduceCluster();
422 }
423 }
424