1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNull;
25 import static org.junit.Assert.assertTrue;
26 import static org.junit.Assert.fail;
27
28 import java.io.IOException;
29 import java.lang.reflect.Field;
30 import java.lang.reflect.Modifier;
31 import java.net.SocketTimeoutException;
32 import java.util.ArrayList;
33 import java.util.HashMap;
34 import java.util.List;
35 import java.util.Map;
36 import java.util.Random;
37 import java.util.concurrent.ExecutorService;
38 import java.util.concurrent.SynchronousQueue;
39 import java.util.concurrent.ThreadPoolExecutor;
40 import java.util.concurrent.TimeUnit;
41 import java.util.concurrent.atomic.AtomicBoolean;
42 import java.util.concurrent.atomic.AtomicInteger;
43 import java.util.concurrent.atomic.AtomicLong;
44 import java.util.concurrent.atomic.AtomicReference;
45
46 import org.apache.commons.logging.Log;
47 import org.apache.commons.logging.LogFactory;
48 import org.apache.hadoop.conf.Configuration;
49 import org.apache.hadoop.hbase.Cell;
50 import org.apache.hadoop.hbase.HBaseConfiguration;
51 import org.apache.hadoop.hbase.HBaseTestingUtility;
52 import org.apache.hadoop.hbase.HConstants;
53 import org.apache.hadoop.hbase.HRegionLocation;
54 import org.apache.hadoop.hbase.HTableDescriptor;
55 import org.apache.hadoop.hbase.RegionLocations;
56 import org.apache.hadoop.hbase.ServerName;
57 import org.apache.hadoop.hbase.TableName;
58 import org.apache.hadoop.hbase.Waiter;
59 import org.apache.hadoop.hbase.client.ConnectionManager.HConnectionImplementation;
60 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
61 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
62 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
63 import org.apache.hadoop.hbase.exceptions.ClientExceptionsUtil;
64 import org.apache.hadoop.hbase.exceptions.DeserializationException;
65 import org.apache.hadoop.hbase.exceptions.RegionMovedException;
66 import org.apache.hadoop.hbase.filter.Filter;
67 import org.apache.hadoop.hbase.filter.FilterBase;
68 import org.apache.hadoop.hbase.ipc.RpcClient;
69 import org.apache.hadoop.hbase.master.HMaster;
70 import org.apache.hadoop.hbase.regionserver.HRegionServer;
71 import org.apache.hadoop.hbase.regionserver.Region;
72 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
73 import org.apache.hadoop.hbase.testclassification.MediumTests;
74 import org.apache.hadoop.hbase.util.Bytes;
75 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
76 import org.apache.hadoop.hbase.util.JVMClusterUtil;
77 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
78 import org.apache.hadoop.hbase.util.Threads;
79 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
80 import org.junit.AfterClass;
81 import org.junit.Assert;
82 import org.junit.BeforeClass;
83 import org.junit.Ignore;
84 import org.junit.Test;
85 import org.junit.experimental.categories.Category;
86
87 import com.google.common.collect.Lists;
88
89
90
91
92 @Category(MediumTests.class)
93 public class TestHCM {
94 private static final Log LOG = LogFactory.getLog(TestHCM.class);
95 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
96 private static final TableName TABLE_NAME =
97 TableName.valueOf("test");
98 private static final TableName TABLE_NAME1 =
99 TableName.valueOf("test1");
100 private static final TableName TABLE_NAME2 =
101 TableName.valueOf("test2");
102 private static final TableName TABLE_NAME3 =
103 TableName.valueOf("test3");
104 private static final TableName TABLE_NAME4 =
105 TableName.valueOf("test4");
106 private static final byte[] FAM_NAM = Bytes.toBytes("f");
107 private static final byte[] ROW = Bytes.toBytes("bbb");
108 private static final byte[] ROW_X = Bytes.toBytes("xxx");
109 private static Random _randy = new Random();
110
111
112
113
114 public static class SleepAndFailFirstTime extends BaseRegionObserver {
115 static final AtomicLong ct = new AtomicLong(0);
116
117 public SleepAndFailFirstTime() {
118 }
119
120 @Override
121 public void preGetOp(final ObserverContext<RegionCoprocessorEnvironment> e,
122 final Get get, final List<Cell> results) throws IOException {
123 Threads.sleep(20000);
124 if (ct.incrementAndGet() == 1){
125 throw new IOException("first call I fail");
126 }
127 }
128 }
129
130 public static class SleepCoprocessor extends BaseRegionObserver {
131 public static final int SLEEP_TIME = 5000;
132
133 @Override public void preGetOp(final ObserverContext<RegionCoprocessorEnvironment> e,
134 final Get get, final List<Cell> results) throws IOException {
135 Threads.sleep(SLEEP_TIME);
136 }
137 }
138
139 @BeforeClass
140 public static void setUpBeforeClass() throws Exception {
141 TEST_UTIL.getConfiguration().setBoolean(HConstants.STATUS_PUBLISHED, true);
142 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
143 TEST_UTIL.startMiniCluster(2);
144 }
145
146 @AfterClass public static void tearDownAfterClass() throws Exception {
147 TEST_UTIL.shutdownMiniCluster();
148 }
149
150
151 private static int getHConnectionManagerCacheSize(){
152 return HConnectionTestingUtility.getConnectionCount();
153 }
154
155 @Test
156 public void testClusterConnection() throws IOException {
157 ThreadPoolExecutor otherPool = new ThreadPoolExecutor(1, 1,
158 5, TimeUnit.SECONDS,
159 new SynchronousQueue<Runnable>(),
160 Threads.newDaemonThreadFactory("test-hcm"));
161
162 HConnection con1 = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
163 HConnection con2 = HConnectionManager.createConnection(TEST_UTIL.getConfiguration(), otherPool);
164
165 assertTrue(otherPool == ((HConnectionImplementation)con2).getCurrentBatchPool());
166
167 String tableName = "testClusterConnection";
168 TEST_UTIL.createTable(tableName.getBytes(), FAM_NAM).close();
169 HTable t = (HTable)con1.getTable(tableName, otherPool);
170
171 assertNull("Internal Thread pool should be null", ((HConnectionImplementation)con1).getCurrentBatchPool());
172
173 assertTrue(otherPool == t.getPool());
174 t.close();
175
176 t = (HTable)con2.getTable(tableName);
177
178 assertTrue(otherPool == t.getPool());
179 t.close();
180
181 t = (HTable)con2.getTable(Bytes.toBytes(tableName));
182
183 assertTrue(otherPool == t.getPool());
184 t.close();
185
186 t = (HTable)con2.getTable(TableName.valueOf(tableName));
187
188 assertTrue(otherPool == t.getPool());
189 t.close();
190
191 t = (HTable)con1.getTable(tableName);
192 ExecutorService pool = ((HConnectionImplementation)con1).getCurrentBatchPool();
193
194 assertNotNull("An internal Thread pool should have been created", pool);
195
196 assertTrue(t.getPool() == pool);
197 t.close();
198
199 t = (HTable)con1.getTable(tableName);
200
201 assertTrue(t.getPool() == pool);
202 t.close();
203
204 con1.close();
205
206 assertTrue(pool.isShutdown());
207
208 con2.close();
209
210 assertFalse(otherPool.isShutdown());
211 otherPool.shutdownNow();
212 }
213
214
215
216
217
218 @Test
219 public void testAdminFactory() throws IOException {
220 Connection con1 = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
221 Admin admin = con1.getAdmin();
222 assertTrue(admin.getConnection() == con1);
223 assertTrue(admin.getConfiguration() == TEST_UTIL.getConfiguration());
224 con1.close();
225 }
226
227
228 @Ignore @Test(expected = RegionServerStoppedException.class)
229 public void testClusterStatus() throws Exception {
230
231 TableName tn =
232 TableName.valueOf("testClusterStatus");
233 byte[] cf = "cf".getBytes();
234 byte[] rk = "rk1".getBytes();
235
236 JVMClusterUtil.RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
237 rs.waitForServerOnline();
238 final ServerName sn = rs.getRegionServer().getServerName();
239
240 HTable t = TEST_UTIL.createTable(tn, cf);
241 TEST_UTIL.waitTableAvailable(tn);
242
243 while(TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
244 getRegionStates().isRegionsInTransition()){
245 Thread.sleep(1);
246 }
247 final HConnectionImplementation hci = (HConnectionImplementation)t.getConnection();
248 while (t.getRegionLocation(rk).getPort() != sn.getPort()){
249 TEST_UTIL.getHBaseAdmin().move(t.getRegionLocation(rk).getRegionInfo().
250 getEncodedNameAsBytes(), Bytes.toBytes(sn.toString()));
251 while(TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
252 getRegionStates().isRegionsInTransition()){
253 Thread.sleep(1);
254 }
255 hci.clearRegionCache(tn);
256 }
257 Assert.assertNotNull(hci.clusterStatusListener);
258 TEST_UTIL.assertRegionOnServer(t.getRegionLocation(rk).getRegionInfo(), sn, 20000);
259
260 Put p1 = new Put(rk);
261 p1.add(cf, "qual".getBytes(), "val".getBytes());
262 t.put(p1);
263
264 rs.getRegionServer().abort("I'm dead");
265
266
267 TEST_UTIL.waitFor(40000, 1000, true, new Waiter.Predicate<Exception>() {
268 @Override
269 public boolean evaluate() throws Exception {
270 return TEST_UTIL.getHBaseCluster().getMaster().getServerManager().
271 getDeadServers().isDeadServer(sn);
272 }
273 });
274
275 TEST_UTIL.waitFor(40000, 1000, true, new Waiter.Predicate<Exception>() {
276 @Override
277 public boolean evaluate() throws Exception {
278 return hci.clusterStatusListener.isDeadServer(sn);
279 }
280 });
281
282 t.close();
283 hci.getClient(sn);
284 }
285
286
287
288
289
290 @Test
291 public void testConnectionCloseAllowsInterrupt() throws Exception {
292 testConnectionClose(true);
293 }
294
295 @Test
296 public void testConnectionNotAllowsInterrupt() throws Exception {
297 testConnectionClose(false);
298 }
299
300
301
302
303
304
305
306
307
308 @Test
309 public void testOperationTimeout() throws Exception {
310 HTableDescriptor hdt = TEST_UTIL.createTableDescriptor("HCM-testOperationTimeout");
311 hdt.addCoprocessor(SleepAndFailFirstTime.class.getName());
312 HTable table = TEST_UTIL.createTable(hdt, new byte[][]{FAM_NAM}, TEST_UTIL.getConfiguration());
313 table.setRpcTimeout(Integer.MAX_VALUE);
314
315 table.setOperationTimeout(120 * 1000);
316 table.get(new Get(FAM_NAM));
317
318
319 SleepAndFailFirstTime.ct.set(0);
320 try {
321 table.setOperationTimeout(30 * 1000);
322 table.get(new Get(FAM_NAM));
323 Assert.fail("We expect an exception here");
324 } catch (SocketTimeoutException e) {
325
326
327
328 LOG.info("We received an exception, as expected ", e);
329 } catch (IOException e) {
330 Assert.fail("Wrong exception:" + e.getMessage());
331 } finally {
332 table.close();
333 }
334 }
335
336 @Test(expected = RetriesExhaustedException.class)
337 public void testRpcTimeout() throws Exception {
338 HTableDescriptor hdt = TEST_UTIL.createTableDescriptor("HCM-testRpcTimeout");
339 hdt.addCoprocessor(SleepCoprocessor.class.getName());
340 Configuration c = new Configuration(TEST_UTIL.getConfiguration());
341
342 try (Table t = TEST_UTIL.createTable(hdt, new byte[][] { FAM_NAM }, c)) {
343 assert t instanceof HTable;
344 HTable table = (HTable) t;
345 table.setRpcTimeout(SleepCoprocessor.SLEEP_TIME / 2);
346 table.setOperationTimeout(SleepCoprocessor.SLEEP_TIME * 100);
347 table.get(new Get(FAM_NAM));
348 }
349 }
350
351 private void testConnectionClose(boolean allowsInterrupt) throws Exception {
352 TableName tableName = TableName.valueOf("HCM-testConnectionClose" + allowsInterrupt);
353 TEST_UTIL.createTable(tableName, FAM_NAM).close();
354
355 boolean previousBalance = TEST_UTIL.getHBaseAdmin().setBalancerRunning(false, true);
356
357 Configuration c2 = new Configuration(TEST_UTIL.getConfiguration());
358
359 c2.set(HConstants.HBASE_CLIENT_INSTANCE_ID, String.valueOf(-1));
360 c2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 100);
361 c2.setInt(HConstants.HBASE_CLIENT_PAUSE, 0);
362 c2.setInt(RpcClient.FAILED_SERVER_EXPIRY_KEY, 0);
363 c2.setBoolean(RpcClient.SPECIFIC_WRITE_THREAD, allowsInterrupt);
364
365 final HTable table = new HTable(c2, tableName);
366
367 Put put = new Put(ROW);
368 put.add(FAM_NAM, ROW, ROW);
369 table.put(put);
370
371
372 final AtomicInteger step = new AtomicInteger(0);
373
374 final AtomicReference<Throwable> failed = new AtomicReference<Throwable>(null);
375 Thread t = new Thread("testConnectionCloseThread") {
376 @Override
377 public void run() {
378 int done = 0;
379 try {
380 step.set(1);
381 while (step.get() == 1) {
382 Get get = new Get(ROW);
383 table.get(get);
384 done++;
385 if (done % 100 == 0)
386 LOG.info("done=" + done);
387 }
388 } catch (Throwable t) {
389 failed.set(t);
390 LOG.error(t);
391 }
392 step.set(3);
393 }
394 };
395 t.start();
396 TEST_UTIL.waitFor(20000, new Waiter.Predicate<Exception>() {
397 @Override
398 public boolean evaluate() throws Exception {
399 return step.get() == 1;
400 }
401 });
402
403 ServerName sn = table.getRegionLocation(ROW).getServerName();
404 ConnectionManager.HConnectionImplementation conn =
405 (ConnectionManager.HConnectionImplementation) table.getConnection();
406 RpcClient rpcClient = conn.getRpcClient();
407
408 LOG.info("Going to cancel connections. connection=" + conn.toString() + ", sn=" + sn);
409 for (int i = 0; i < 5000; i++) {
410 rpcClient.cancelConnections(sn);
411 Thread.sleep(5);
412 }
413
414 step.compareAndSet(1, 2);
415
416
417 TEST_UTIL.waitFor(40000, new Waiter.Predicate<Exception>() {
418 @Override
419 public boolean evaluate() throws Exception {
420 return step.get() == 3;
421 }
422 });
423 table.close();
424 Assert.assertTrue("Unexpected exception is " + failed.get(), failed.get() == null);
425 TEST_UTIL.getHBaseAdmin().setBalancerRunning(previousBalance, true);
426 }
427
428
429
430
431 @Test
432 public void testConnectionIdle() throws Exception {
433 TableName tableName = TableName.valueOf("HCM-testConnectionIdle");
434 TEST_UTIL.createTable(tableName, FAM_NAM).close();
435 int idleTime = 20000;
436 boolean previousBalance = TEST_UTIL.getHBaseAdmin().setBalancerRunning(false, true);
437
438 Configuration c2 = new Configuration(TEST_UTIL.getConfiguration());
439
440 c2.set(HConstants.HBASE_CLIENT_INSTANCE_ID, String.valueOf(-1));
441 c2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
442 c2.setInt(RpcClient.IDLE_TIME, idleTime);
443
444 final Table table = new HTable(c2, tableName);
445
446 Put put = new Put(ROW);
447 put.add(FAM_NAM, ROW, ROW);
448 table.put(put);
449
450 ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
451 mee.setValue(System.currentTimeMillis());
452 EnvironmentEdgeManager.injectEdge(mee);
453 LOG.info("first get");
454 table.get(new Get(ROW));
455
456 LOG.info("first get - changing the time & sleeping");
457 mee.incValue(idleTime + 1000);
458 Thread.sleep(1500);
459
460
461 LOG.info("second get - connection has been marked idle in the middle");
462
463
464 table.get(new Get(ROW));
465 mee.incValue(idleTime + 1000);
466
467 LOG.info("third get - connection is idle, but the reader doesn't know yet");
468
469
470
471
472
473
474
475 table.get(new Get(ROW));
476
477 LOG.info("we're done - time will change back");
478
479 table.close();
480 EnvironmentEdgeManager.reset();
481 TEST_UTIL.getHBaseAdmin().setBalancerRunning(previousBalance, true);
482 }
483
484
485
486
487
488
489 @Test
490 public void testConnectionCut() throws Exception {
491
492 TableName tableName = TableName.valueOf("HCM-testConnectionCut");
493
494 TEST_UTIL.createTable(tableName, FAM_NAM).close();
495 boolean previousBalance = TEST_UTIL.getHBaseAdmin().setBalancerRunning(false, true);
496
497 Configuration c2 = new Configuration(TEST_UTIL.getConfiguration());
498
499 c2.set(HConstants.HBASE_CLIENT_INSTANCE_ID, String.valueOf(-1));
500 c2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
501 c2.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 30 * 1000);
502
503 HTable table = new HTable(c2, tableName);
504
505 Put p = new Put(FAM_NAM);
506 p.add(FAM_NAM, FAM_NAM, FAM_NAM);
507 table.put(p);
508
509 final HConnectionImplementation hci = (HConnectionImplementation)table.getConnection();
510 final HRegionLocation loc = table.getRegionLocation(FAM_NAM);
511
512 Get get = new Get(FAM_NAM);
513 Assert.assertNotNull(table.get(get));
514
515 get = new Get(FAM_NAM);
516 get.setFilter(new BlockingFilter());
517
518
519 Thread t = new Thread() {
520 @Override
521 public void run() {
522 synchronized (syncBlockingFilter) {
523 try {
524 syncBlockingFilter.wait();
525 } catch (InterruptedException e) {
526 throw new RuntimeException(e);
527 }
528 }
529 hci.clusterStatusListener.deadServerHandler.newDead(loc.getServerName());
530 }
531 };
532
533 t.start();
534 try {
535 table.get(get);
536 Assert.fail();
537 } catch (IOException expected) {
538 LOG.debug("Received: " + expected);
539 Assert.assertFalse(expected instanceof SocketTimeoutException);
540 Assert.assertFalse(syncBlockingFilter.get());
541 } finally {
542 syncBlockingFilter.set(true);
543 t.join();
544 HConnectionManager.getConnection(c2).close();
545 TEST_UTIL.getHBaseAdmin().setBalancerRunning(previousBalance, true);
546 }
547
548 table.close();
549 }
550
551 protected static final AtomicBoolean syncBlockingFilter = new AtomicBoolean(false);
552
553 public static class BlockingFilter extends FilterBase {
554 @Override
555 public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
556 int i = 0;
557 while (i++ < 1000 && !syncBlockingFilter.get()) {
558 synchronized (syncBlockingFilter) {
559 syncBlockingFilter.notifyAll();
560 }
561 Threads.sleep(100);
562 }
563 syncBlockingFilter.set(true);
564 return false;
565 }
566 @Override
567 public ReturnCode filterKeyValue(Cell ignored) throws IOException {
568 return ReturnCode.INCLUDE;
569 }
570
571 public static Filter parseFrom(final byte [] pbBytes) throws DeserializationException{
572 return new BlockingFilter();
573 }
574 }
575
576 @Test
577 public void abortingHConnectionRemovesItselfFromHCM() throws Exception {
578
579 Map<HConnectionKey, HConnectionImplementation> oldHBaseInstances =
580 new HashMap<HConnectionKey, HConnectionImplementation>();
581 oldHBaseInstances.putAll(ConnectionManager.CONNECTION_INSTANCES);
582
583 ConnectionManager.CONNECTION_INSTANCES.clear();
584
585 try {
586 HConnection connection = HConnectionManager.getConnection(TEST_UTIL.getConfiguration());
587 connection.abort("test abortingHConnectionRemovesItselfFromHCM", new Exception(
588 "test abortingHConnectionRemovesItselfFromHCM"));
589 Assert.assertNotSame(connection,
590 HConnectionManager.getConnection(TEST_UTIL.getConfiguration()));
591 } finally {
592
593 ConnectionManager.CONNECTION_INSTANCES.clear();
594 ConnectionManager.CONNECTION_INSTANCES.putAll(oldHBaseInstances);
595 }
596 }
597
598
599
600
601
602
603 @Test
604 public void testRegionCaching() throws Exception{
605 TEST_UTIL.createMultiRegionTable(TABLE_NAME, FAM_NAM).close();
606 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
607 conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
608 HTable table = new HTable(conf, TABLE_NAME);
609
610 TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
611 Put put = new Put(ROW);
612 put.add(FAM_NAM, ROW, ROW);
613 table.put(put);
614 ConnectionManager.HConnectionImplementation conn =
615 (ConnectionManager.HConnectionImplementation)table.getConnection();
616
617 assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
618
619 final int nextPort = conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort() + 1;
620 HRegionLocation loc = conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation();
621 conn.updateCachedLocation(loc.getRegionInfo(), loc.getServerName(),
622 ServerName.valueOf("127.0.0.1", nextPort,
623 HConstants.LATEST_TIMESTAMP), HConstants.LATEST_TIMESTAMP);
624 Assert.assertEquals(conn.getCachedLocation(TABLE_NAME, ROW)
625 .getRegionLocation().getPort(), nextPort);
626
627 conn.clearRegionCache(TABLE_NAME, ROW.clone());
628 RegionLocations rl = conn.getCachedLocation(TABLE_NAME, ROW);
629 assertNull("What is this location?? " + rl, rl);
630
631
632
633 conn.clearRegionCache(TABLE_NAME);
634 Assert.assertEquals(0, conn.getNumberOfCachedRegionLocations(TABLE_NAME));
635 Put put2 = new Put(ROW);
636 put2.add(FAM_NAM, ROW, ROW);
637 table.put(put2);
638 assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
639 assertNotNull(conn.getCachedLocation(TableName.valueOf(TABLE_NAME.getName()), ROW.clone()));
640
641 TEST_UTIL.getHBaseAdmin().setBalancerRunning(false, false);
642 HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
643
644
645 while (master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
646 Thread.sleep(1);
647 }
648
649
650 HRegionLocation toMove = conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation();
651 byte[] regionName = toMove.getRegionInfo().getRegionName();
652 byte[] encodedRegionNameBytes = toMove.getRegionInfo().getEncodedNameAsBytes();
653
654
655 int curServerId = TEST_UTIL.getHBaseCluster().getServerWith(regionName);
656 int destServerId = (curServerId == 0 ? 1 : 0);
657
658 HRegionServer curServer = TEST_UTIL.getHBaseCluster().getRegionServer(curServerId);
659 HRegionServer destServer = TEST_UTIL.getHBaseCluster().getRegionServer(destServerId);
660
661 ServerName destServerName = destServer.getServerName();
662
663
664 Assert.assertTrue(curServer != destServer);
665 Assert.assertFalse(curServer.getServerName().equals(destServer.getServerName()));
666 Assert.assertFalse( toMove.getPort() == destServerName.getPort());
667 Assert.assertNotNull(curServer.getOnlineRegion(regionName));
668 Assert.assertNull(destServer.getOnlineRegion(regionName));
669 Assert.assertFalse(TEST_UTIL.getMiniHBaseCluster().getMaster().
670 getAssignmentManager().getRegionStates().isRegionsInTransition());
671
672
673
674 LOG.info("Move starting region="+toMove.getRegionInfo().getRegionNameAsString());
675 TEST_UTIL.getHBaseAdmin().move(
676 toMove.getRegionInfo().getEncodedNameAsBytes(),
677 destServerName.getServerName().getBytes()
678 );
679
680 while (destServer.getOnlineRegion(regionName) == null ||
681 destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) ||
682 curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) ||
683 master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
684
685 Thread.sleep(1);
686 }
687
688 LOG.info("Move finished for region="+toMove.getRegionInfo().getRegionNameAsString());
689
690
691 Assert.assertNull(curServer.getOnlineRegion(regionName));
692 Assert.assertNotNull(destServer.getOnlineRegion(regionName));
693 Assert.assertFalse(destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
694 Assert.assertFalse(curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
695
696
697
698 Assert.assertFalse(
699 conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation()
700 .getPort() == destServerName.getPort());
701
702
703
704 LOG.info("Put starting");
705 Put put3 = new Put(ROW);
706 put3.add(FAM_NAM, ROW, ROW);
707 try {
708 table.put(put3);
709 Assert.fail("Unreachable point");
710 } catch (RetriesExhaustedWithDetailsException e){
711 LOG.info("Put done, exception caught: " + e.getClass());
712 Assert.assertEquals(1, e.getNumExceptions());
713 Assert.assertEquals(1, e.getCauses().size());
714 Assert.assertArrayEquals(e.getRow(0).getRow(), ROW);
715
716
717 Throwable cause = ClientExceptionsUtil.findException(e.getCause(0));
718 Assert.assertNotNull(cause);
719 Assert.assertTrue(cause instanceof RegionMovedException);
720 }
721 Assert.assertNotNull("Cached connection is null", conn.getCachedLocation(TABLE_NAME, ROW));
722 Assert.assertEquals(
723 "Previous server was " + curServer.getServerName().getHostAndPort(),
724 destServerName.getPort(),
725 conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort());
726
727 Assert.assertFalse(destServer.getRegionsInTransitionInRS()
728 .containsKey(encodedRegionNameBytes));
729 Assert.assertFalse(curServer.getRegionsInTransitionInRS()
730 .containsKey(encodedRegionNameBytes));
731
732
733 LOG.info("Move starting region=" + toMove.getRegionInfo().getRegionNameAsString());
734 TEST_UTIL.getHBaseAdmin().move(
735 toMove.getRegionInfo().getEncodedNameAsBytes(),
736 curServer.getServerName().getServerName().getBytes()
737 );
738
739 while (curServer.getOnlineRegion(regionName) == null ||
740 destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) ||
741 curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) ||
742 master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
743
744 Thread.sleep(1);
745 }
746
747
748 Assert.assertNotNull(curServer.getOnlineRegion(regionName));
749 Assert.assertNull(destServer.getOnlineRegion(regionName));
750 LOG.info("Move finished for region=" + toMove.getRegionInfo().getRegionNameAsString());
751
752
753 Assert.assertFalse(conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort() ==
754 curServer.getServerName().getPort());
755
756 Scan sc = new Scan();
757 sc.setStopRow(ROW);
758 sc.setStartRow(ROW);
759
760
761
762 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
763
764 try {
765 ResultScanner rs = table.getScanner(sc);
766 while (rs.next() != null) {
767 }
768 Assert.fail("Unreachable point");
769 } catch (RetriesExhaustedException e) {
770 LOG.info("Scan done, expected exception caught: " + e.getClass());
771 }
772
773
774 Assert.assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
775 Assert.assertEquals(
776 "Previous server was "+destServer.getServerName().getHostAndPort(),
777 curServer.getServerName().getPort(),
778 conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort());
779
780 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
781 HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
782 table.close();
783 }
784
785
786
787
788
789 @Test
790 public void testConnectionManagement() throws Exception{
791 Table table0 = TEST_UTIL.createTable(TABLE_NAME1, FAM_NAM);
792 Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
793 HTable table = (HTable) conn.getTable(TABLE_NAME1);
794 table.close();
795 assertFalse(conn.isClosed());
796 assertFalse(table.getPool().isShutdown());
797 table = (HTable) conn.getTable(TABLE_NAME1);
798 table.close();
799 assertFalse(table.getPool().isShutdown());
800 conn.close();
801 assertTrue(table.getPool().isShutdown());
802 table0.close();
803 }
804
805
806
807
808 @Test(timeout = 60000)
809 public void testCacheSeqNums() throws Exception{
810 HTable table = TEST_UTIL.createMultiRegionTable(TABLE_NAME2, FAM_NAM);
811 Put put = new Put(ROW);
812 put.add(FAM_NAM, ROW, ROW);
813 table.put(put);
814 ConnectionManager.HConnectionImplementation conn =
815 (ConnectionManager.HConnectionImplementation)table.getConnection();
816
817 HRegionLocation location = conn.getCachedLocation(TABLE_NAME2, ROW).getRegionLocation();
818 assertNotNull(location);
819
820 ServerName anySource = ServerName.valueOf(location.getHostname(), location.getPort() - 1, 0L);
821
822
823 int nextPort = location.getPort() + 1;
824 conn.updateCachedLocation(location.getRegionInfo(), location.getServerName(),
825 ServerName.valueOf("127.0.0.1", nextPort, 0), location.getSeqNum() - 1);
826 location = conn.getCachedLocation(TABLE_NAME2, ROW).getRegionLocation();
827 Assert.assertEquals(nextPort, location.getPort());
828
829
830 nextPort = location.getPort() + 1;
831 conn.updateCachedLocation(location.getRegionInfo(), location.getServerName(),
832 ServerName.valueOf("127.0.0.1", nextPort, 0), location.getSeqNum() - 1);
833 location = conn.getCachedLocation(TABLE_NAME2, ROW).getRegionLocation();
834 Assert.assertEquals(nextPort, location.getPort());
835
836
837 nextPort = location.getPort() + 1;
838 conn.updateCachedLocation(location.getRegionInfo(), anySource,
839 ServerName.valueOf("127.0.0.1", nextPort, 0), location.getSeqNum() + 1);
840 location = conn.getCachedLocation(TABLE_NAME2, ROW).getRegionLocation();
841 Assert.assertEquals(nextPort, location.getPort());
842
843
844 nextPort = location.getPort() + 1;
845 conn.updateCachedLocation(location.getRegionInfo(), anySource,
846 ServerName.valueOf("127.0.0.1", nextPort, 0), location.getSeqNum() - 1);
847 location = conn.getCachedLocation(TABLE_NAME2, ROW).getRegionLocation();
848 Assert.assertEquals(nextPort - 1, location.getPort());
849 table.close();
850 }
851
852
853
854
855
856 @Test
857 public void testConnectionSameness() throws Exception {
858 Connection previousConnection = null;
859 for (int i = 0; i < 2; i++) {
860
861 Configuration configuration = TEST_UTIL.getConfiguration();
862 configuration.set("some_key", String.valueOf(_randy.nextInt()));
863 LOG.info("The hash code of the current configuration is: "
864 + configuration.hashCode());
865 Connection currentConnection = HConnectionManager
866 .getConnection(configuration);
867 if (previousConnection != null) {
868 assertTrue(
869 "Did not get the same connection even though its key didn't change",
870 previousConnection == currentConnection);
871 }
872 previousConnection = currentConnection;
873
874
875
876
877 configuration.set("other_key", String.valueOf(_randy.nextInt()));
878 }
879 }
880
881
882
883
884
885
886
887 @Deprecated
888 @Test
889 public void testConnectionUniqueness() throws Exception {
890 int zkmaxconnections = TEST_UTIL.getConfiguration().
891 getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS,
892 HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS);
893
894
895
896 int maxConnections = Math.min(zkmaxconnections - 1, 20);
897 List<HConnection> connections = new ArrayList<HConnection>(maxConnections);
898 Connection previousConnection = null;
899 try {
900 for (int i = 0; i < maxConnections; i++) {
901
902 Configuration configuration = new Configuration(TEST_UTIL.getConfiguration());
903 configuration.set("some_key", String.valueOf(_randy.nextInt()));
904 configuration.set(HConstants.HBASE_CLIENT_INSTANCE_ID,
905 String.valueOf(_randy.nextInt()));
906 LOG.info("The hash code of the current configuration is: "
907 + configuration.hashCode());
908 HConnection currentConnection =
909 HConnectionManager.getConnection(configuration);
910 if (previousConnection != null) {
911 assertTrue("Got the same connection even though its key changed!",
912 previousConnection != currentConnection);
913 }
914
915
916
917
918 configuration.set("other_key", String.valueOf(_randy.nextInt()));
919
920 previousConnection = currentConnection;
921 LOG.info("The current HConnectionManager#HBASE_INSTANCES cache size is: "
922 + getHConnectionManagerCacheSize());
923 Thread.sleep(50);
924 connections.add(currentConnection);
925 }
926 } finally {
927 for (Connection c: connections) {
928
929 HConnectionManager.deleteConnection(c.getConfiguration());
930 }
931 }
932 }
933
934 @Test
935 public void testClosing() throws Exception {
936 Configuration configuration =
937 new Configuration(TEST_UTIL.getConfiguration());
938 configuration.set(HConstants.HBASE_CLIENT_INSTANCE_ID,
939 String.valueOf(_randy.nextInt()));
940
941 Connection c1 = ConnectionFactory.createConnection(configuration);
942
943 Connection c2 = ConnectionFactory.createConnection(configuration);
944
945 Connection c3 = HConnectionManager.getConnection(configuration);
946 Connection c4 = HConnectionManager.getConnection(configuration);
947 assertTrue(c3 == c4);
948
949 c1.close();
950 assertTrue(c1.isClosed());
951 assertFalse(c2.isClosed());
952 assertFalse(c3.isClosed());
953
954 c3.close();
955
956 assertFalse(c3.isClosed());
957 c3.close();
958 assertTrue(c3.isClosed());
959
960 Connection c5 = HConnectionManager.getConnection(configuration);
961 assertTrue(c5 != c3);
962
963 assertFalse(c2.isClosed());
964 c2.close();
965 assertTrue(c2.isClosed());
966 c5.close();
967 assertTrue(c5.isClosed());
968 }
969
970
971
972
973
974 @Test
975 public void testCreateConnection() throws Exception {
976 Configuration configuration = TEST_UTIL.getConfiguration();
977 Connection c1 = ConnectionFactory.createConnection(configuration);
978 Connection c2 = ConnectionFactory.createConnection(configuration);
979
980 assertTrue(c1 != c2);
981 assertTrue(c1.getConfiguration() == c2.getConfiguration());
982
983 Connection c3 = HConnectionManager.getConnection(configuration);
984 assertTrue(c1 != c3);
985 assertTrue(c2 != c3);
986 }
987
988
989
990
991
992
993
994 @Test(timeout = 60000)
995 public void testConnection() throws Exception{
996
997 Configuration c = new Configuration();
998 c.set(HConstants.ZOOKEEPER_QUORUM,
999 TEST_UTIL.getConfiguration().get(HConstants.ZOOKEEPER_QUORUM));
1000 c.set(HConstants.ZOOKEEPER_CLIENT_PORT ,
1001 TEST_UTIL.getConfiguration().get(HConstants.ZOOKEEPER_CLIENT_PORT));
1002
1003
1004 HConnection conn = HConnectionManager.getConnection(c);
1005 assertTrue( conn.isMasterRunning() );
1006 conn.close();
1007 }
1008
1009 private int setNumTries(HConnectionImplementation hci, int newVal) throws Exception {
1010 Field numTries = hci.getClass().getDeclaredField("numTries");
1011 numTries.setAccessible(true);
1012 Field modifiersField = Field.class.getDeclaredField("modifiers");
1013 modifiersField.setAccessible(true);
1014 modifiersField.setInt(numTries, numTries.getModifiers() & ~Modifier.FINAL);
1015 final int prevNumRetriesVal = (Integer)numTries.get(hci);
1016 numTries.set(hci, newVal);
1017
1018 return prevNumRetriesVal;
1019 }
1020
1021 @Test (timeout=30000)
1022 public void testMulti() throws Exception {
1023 HTable table = TEST_UTIL.createMultiRegionTable(TABLE_NAME3, FAM_NAM);
1024 try {
1025 ConnectionManager.HConnectionImplementation conn =
1026 ( ConnectionManager.HConnectionImplementation)table.getConnection();
1027
1028
1029
1030 conn.clearRegionCache(TABLE_NAME3);
1031 Assert.assertEquals(0, conn.getNumberOfCachedRegionLocations(TABLE_NAME3));
1032
1033 TEST_UTIL.getHBaseAdmin().setBalancerRunning(false, false);
1034 HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
1035
1036
1037 while (master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
1038 Thread.sleep(1);
1039 }
1040
1041 Put put = new Put(ROW_X);
1042 put.add(FAM_NAM, ROW_X, ROW_X);
1043 table.put(put);
1044
1045
1046 HRegionLocation toMove = conn.getCachedLocation(TABLE_NAME3, ROW_X).getRegionLocation();
1047 byte[] regionName = toMove.getRegionInfo().getRegionName();
1048 byte[] encodedRegionNameBytes = toMove.getRegionInfo().getEncodedNameAsBytes();
1049
1050
1051 int curServerId = TEST_UTIL.getHBaseCluster().getServerWith(regionName);
1052 int destServerId = (curServerId == 0 ? 1 : 0);
1053
1054 HRegionServer curServer = TEST_UTIL.getHBaseCluster().getRegionServer(curServerId);
1055 HRegionServer destServer = TEST_UTIL.getHBaseCluster().getRegionServer(destServerId);
1056
1057 ServerName destServerName = destServer.getServerName();
1058
1059
1060 List<Region> regions = curServer.getOnlineRegions(TABLE_NAME3);
1061 byte[] otherRow = null;
1062 for (Region region : regions) {
1063 if (!region.getRegionInfo().getEncodedName().equals(toMove.getRegionInfo().getEncodedName())
1064 && Bytes.BYTES_COMPARATOR.compare(region.getRegionInfo().getStartKey(), ROW_X) < 0) {
1065 otherRow = region.getRegionInfo().getStartKey();
1066 break;
1067 }
1068 }
1069 assertNotNull(otherRow);
1070
1071 if (otherRow.length <= 0) otherRow = Bytes.toBytes("aaa");
1072 Put put2 = new Put(otherRow);
1073 put2.add(FAM_NAM, otherRow, otherRow);
1074 table.put(put2);
1075
1076
1077 Assert.assertTrue(curServer != destServer);
1078 Assert.assertNotEquals(curServer.getServerName(), destServer.getServerName());
1079 Assert.assertNotEquals(toMove.getPort(), destServerName.getPort());
1080 Assert.assertNotNull(curServer.getOnlineRegion(regionName));
1081 Assert.assertNull(destServer.getOnlineRegion(regionName));
1082 Assert.assertFalse(TEST_UTIL.getMiniHBaseCluster().getMaster().
1083 getAssignmentManager().getRegionStates().isRegionsInTransition());
1084
1085
1086
1087 LOG.info("Move starting region="+toMove.getRegionInfo().getRegionNameAsString());
1088 TEST_UTIL.getHBaseAdmin().move(
1089 toMove.getRegionInfo().getEncodedNameAsBytes(),
1090 destServerName.getServerName().getBytes()
1091 );
1092
1093 while (destServer.getOnlineRegion(regionName) == null ||
1094 destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) ||
1095 curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) ||
1096 master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
1097
1098 Thread.sleep(1);
1099 }
1100
1101 LOG.info("Move finished for region="+toMove.getRegionInfo().getRegionNameAsString());
1102
1103
1104 Assert.assertNull(curServer.getOnlineRegion(regionName));
1105 Assert.assertNotNull(destServer.getOnlineRegion(regionName));
1106 Assert.assertFalse(destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
1107 Assert.assertFalse(curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
1108
1109
1110
1111 Assert.assertFalse(
1112 conn.getCachedLocation(TABLE_NAME3, ROW_X).getRegionLocation()
1113 .getPort() == destServerName.getPort());
1114
1115
1116 final int prevNumRetriesVal = setNumTries(conn, 2);
1117
1118 Put put3 = new Put(ROW_X);
1119 put3.add(FAM_NAM, ROW_X, ROW_X);
1120 Put put4 = new Put(otherRow);
1121 put4.add(FAM_NAM, otherRow, otherRow);
1122
1123
1124 table.batch(Lists.newArrayList(put4, put3));
1125
1126
1127 setNumTries(conn, prevNumRetriesVal);
1128 } finally {
1129 table.close();
1130 }
1131 }
1132
1133 @Ignore ("Test presumes RETRY_BACKOFF will never change; it has") @Test
1134 public void testErrorBackoffTimeCalculation() throws Exception {
1135
1136 final long ANY_PAUSE = 100;
1137 ServerName location = ServerName.valueOf("127.0.0.1", 1, 0);
1138 ServerName diffLocation = ServerName.valueOf("127.0.0.1", 2, 0);
1139
1140 ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge();
1141 EnvironmentEdgeManager.injectEdge(timeMachine);
1142 try {
1143 long timeBase = timeMachine.currentTime();
1144 long largeAmountOfTime = ANY_PAUSE * 1000;
1145 ConnectionManager.ServerErrorTracker tracker =
1146 new ConnectionManager.ServerErrorTracker(largeAmountOfTime, 100);
1147
1148
1149 assertEquals(0, tracker.calculateBackoffTime(location, ANY_PAUSE));
1150
1151
1152 tracker.reportServerError(location);
1153 assertEqualsWithJitter(ANY_PAUSE, tracker.calculateBackoffTime(location, ANY_PAUSE));
1154 tracker.reportServerError(location);
1155 tracker.reportServerError(location);
1156 tracker.reportServerError(location);
1157 assertEqualsWithJitter(ANY_PAUSE * 5, tracker.calculateBackoffTime(location, ANY_PAUSE));
1158
1159
1160 assertEquals(0, tracker.calculateBackoffTime(diffLocation, ANY_PAUSE));
1161 tracker.reportServerError(diffLocation);
1162 assertEqualsWithJitter(ANY_PAUSE, tracker.calculateBackoffTime(diffLocation, ANY_PAUSE));
1163
1164
1165 assertEqualsWithJitter(ANY_PAUSE * 10,
1166 tracker.calculateBackoffTime(location, ANY_PAUSE * 2));
1167
1168
1169
1170 long timeShift = (long)(ANY_PAUSE * 0.5);
1171 timeMachine.setValue(timeBase + timeShift);
1172 assertEqualsWithJitter((ANY_PAUSE * 5) - timeShift,
1173 tracker.calculateBackoffTime(location, ANY_PAUSE), ANY_PAUSE * 2);
1174
1175
1176 timeMachine.setValue(timeBase + ANY_PAUSE * 100);
1177 assertEquals(0, tracker.calculateBackoffTime(location, ANY_PAUSE));
1178
1179
1180 long timeLeft = (long)(ANY_PAUSE * 0.5);
1181 timeMachine.setValue(timeBase + largeAmountOfTime - timeLeft);
1182 assertTrue(tracker.canRetryMore(1));
1183 tracker.reportServerError(location);
1184 assertEquals(timeLeft, tracker.calculateBackoffTime(location, ANY_PAUSE));
1185 timeMachine.setValue(timeBase + largeAmountOfTime);
1186 assertFalse(tracker.canRetryMore(1));
1187 } finally {
1188 EnvironmentEdgeManager.reset();
1189 }
1190 }
1191
1192 private static void assertEqualsWithJitter(long expected, long actual) {
1193 assertEqualsWithJitter(expected, actual, expected);
1194 }
1195
1196 private static void assertEqualsWithJitter(long expected, long actual, long jitterBase) {
1197 assertTrue("Value not within jitter: " + expected + " vs " + actual,
1198 Math.abs(actual - expected) <= (0.01f * jitterBase));
1199 }
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209 @Ignore ("Flakey test: See HBASE-8996")@Test
1210 public void testDeleteForZKConnLeak() throws Exception {
1211 TEST_UTIL.createTable(TABLE_NAME4, FAM_NAM);
1212 final Configuration config = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
1213 config.setInt("zookeeper.recovery.retry", 1);
1214 config.setInt("zookeeper.recovery.retry.intervalmill", 1000);
1215 config.setInt("hbase.rpc.timeout", 2000);
1216 config.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
1217
1218 ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 10,
1219 5, TimeUnit.SECONDS,
1220 new SynchronousQueue<Runnable>(),
1221 Threads.newDaemonThreadFactory("test-hcm-delete"));
1222
1223 pool.submit(new Runnable() {
1224 @Override
1225 public void run() {
1226 while (!Thread.interrupted()) {
1227 try {
1228 HConnection conn = HConnectionManager.getConnection(config);
1229 LOG.info("Connection " + conn);
1230 HConnectionManager.deleteStaleConnection(conn);
1231 LOG.info("Connection closed " + conn);
1232
1233
1234
1235 Threads.sleep(10);
1236 } catch (Exception e) {
1237 }
1238 }
1239 }
1240 });
1241
1242
1243 for (int i = 0; i < 30; i++) {
1244 Connection c1 = null;
1245 try {
1246 c1 = ConnectionManager.getConnectionInternal(config);
1247 LOG.info("HTable connection " + i + " " + c1);
1248 Table table = new HTable(config, TABLE_NAME4, pool);
1249 table.close();
1250 LOG.info("HTable connection " + i + " closed " + c1);
1251 } catch (Exception e) {
1252 LOG.info("We actually want this to happen!!!! So we can see if we are leaking zk", e);
1253 } finally {
1254 if (c1 != null) {
1255 if (c1.isClosed()) {
1256
1257 Field zkwField = c1.getClass().getDeclaredField("keepAliveZookeeper");
1258 zkwField.setAccessible(true);
1259 Object watcher = zkwField.get(c1);
1260
1261 if (watcher != null) {
1262 if (((ZooKeeperWatcher)watcher).getRecoverableZooKeeper().getState().isAlive()) {
1263
1264
1265 Thread.sleep(1000);
1266 if (((ZooKeeperWatcher) watcher).getRecoverableZooKeeper().getState().isAlive()) {
1267 pool.shutdownNow();
1268 fail("Live zookeeper in closed connection");
1269 }
1270 }
1271 }
1272 }
1273 c1.close();
1274 }
1275 }
1276 }
1277 pool.shutdownNow();
1278 }
1279
1280 @Test(timeout = 60000)
1281 public void testConnectionRideOverClusterRestart() throws IOException, InterruptedException {
1282 Configuration config = new Configuration(TEST_UTIL.getConfiguration());
1283
1284 TableName tableName = TableName.valueOf("testConnectionRideOverClusterRestart");
1285 TEST_UTIL.createTable(tableName.getName(), new byte[][] {FAM_NAM}, config).close();
1286
1287 Connection connection = ConnectionFactory.createConnection(config);
1288 Table table = connection.getTable(tableName);
1289
1290
1291 table.get(new Get(Bytes.toBytes("foo")));
1292
1293
1294 TEST_UTIL.shutdownMiniHBaseCluster();
1295 TEST_UTIL.restartHBaseCluster(2);
1296
1297 table.get(new Get(Bytes.toBytes("foo")));
1298 TEST_UTIL.deleteTable(tableName);
1299 table.close();
1300 connection.close();
1301 }
1302 }
1303