View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  import static org.junit.Assert.fail;
27  
28  import java.io.IOException;
29  import java.lang.reflect.Field;
30  import java.lang.reflect.Modifier;
31  import java.net.SocketTimeoutException;
32  import java.util.ArrayList;
33  import java.util.HashMap;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.Random;
37  import java.util.concurrent.ExecutorService;
38  import java.util.concurrent.SynchronousQueue;
39  import java.util.concurrent.ThreadPoolExecutor;
40  import java.util.concurrent.TimeUnit;
41  import java.util.concurrent.atomic.AtomicBoolean;
42  import java.util.concurrent.atomic.AtomicInteger;
43  import java.util.concurrent.atomic.AtomicLong;
44  import java.util.concurrent.atomic.AtomicReference;
45  
46  import org.apache.commons.logging.Log;
47  import org.apache.commons.logging.LogFactory;
48  import org.apache.hadoop.conf.Configuration;
49  import org.apache.hadoop.hbase.Cell;
50  import org.apache.hadoop.hbase.HBaseConfiguration;
51  import org.apache.hadoop.hbase.HBaseTestingUtility;
52  import org.apache.hadoop.hbase.HConstants;
53  import org.apache.hadoop.hbase.HRegionLocation;
54  import org.apache.hadoop.hbase.HTableDescriptor;
55  import org.apache.hadoop.hbase.RegionLocations;
56  import org.apache.hadoop.hbase.ServerName;
57  import org.apache.hadoop.hbase.TableName;
58  import org.apache.hadoop.hbase.Waiter;
59  import org.apache.hadoop.hbase.client.ConnectionManager.HConnectionImplementation;
60  import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
61  import org.apache.hadoop.hbase.coprocessor.ObserverContext;
62  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
63  import org.apache.hadoop.hbase.exceptions.ClientExceptionsUtil;
64  import org.apache.hadoop.hbase.exceptions.DeserializationException;
65  import org.apache.hadoop.hbase.exceptions.RegionMovedException;
66  import org.apache.hadoop.hbase.filter.Filter;
67  import org.apache.hadoop.hbase.filter.FilterBase;
68  import org.apache.hadoop.hbase.ipc.RpcClient;
69  import org.apache.hadoop.hbase.master.HMaster;
70  import org.apache.hadoop.hbase.regionserver.HRegionServer;
71  import org.apache.hadoop.hbase.regionserver.Region;
72  import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
73  import org.apache.hadoop.hbase.testclassification.MediumTests;
74  import org.apache.hadoop.hbase.util.Bytes;
75  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
76  import org.apache.hadoop.hbase.util.JVMClusterUtil;
77  import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
78  import org.apache.hadoop.hbase.util.Threads;
79  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
80  import org.junit.AfterClass;
81  import org.junit.Assert;
82  import org.junit.BeforeClass;
83  import org.junit.Ignore;
84  import org.junit.Test;
85  import org.junit.experimental.categories.Category;
86  
87  import com.google.common.collect.Lists;
88  
89  /**
90   * This class is for testing HBaseConnectionManager features
91   */
92  @Category(MediumTests.class)
93  public class TestHCM {
94    private static final Log LOG = LogFactory.getLog(TestHCM.class);
95    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
96    private static final TableName TABLE_NAME =
97        TableName.valueOf("test");
98    private static final TableName TABLE_NAME1 =
99        TableName.valueOf("test1");
100   private static final TableName TABLE_NAME2 =
101       TableName.valueOf("test2");
102   private static final TableName TABLE_NAME3 =
103       TableName.valueOf("test3");
104   private static final TableName TABLE_NAME4 =
105       TableName.valueOf("test4");
106   private static final byte[] FAM_NAM = Bytes.toBytes("f");
107   private static final byte[] ROW = Bytes.toBytes("bbb");
108   private static final byte[] ROW_X = Bytes.toBytes("xxx");
109   private static Random _randy = new Random();
110 
111 /**
112 * This copro sleeps 20 second. The first call it fails. The second time, it works.
113 */
114   public static class SleepAndFailFirstTime extends BaseRegionObserver {
115     static final AtomicLong ct = new AtomicLong(0);
116 
117     public SleepAndFailFirstTime() {
118     }
119 
120     @Override
121     public void preGetOp(final ObserverContext<RegionCoprocessorEnvironment> e,
122               final Get get, final List<Cell> results) throws IOException {
123       Threads.sleep(20000);
124       if (ct.incrementAndGet() == 1){
125         throw new IOException("first call I fail");
126       }
127     }
128   }
129 
130   public static class SleepCoprocessor extends BaseRegionObserver {
131     public static final int SLEEP_TIME = 5000;
132 
133     @Override public void preGetOp(final ObserverContext<RegionCoprocessorEnvironment> e,
134         final Get get, final List<Cell> results) throws IOException {
135       Threads.sleep(SLEEP_TIME);
136     }
137   }
138 
139   @BeforeClass
140   public static void setUpBeforeClass() throws Exception {
141     TEST_UTIL.getConfiguration().setBoolean(HConstants.STATUS_PUBLISHED, true);
142     TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
143     TEST_UTIL.startMiniCluster(2);
144   }
145 
146   @AfterClass public static void tearDownAfterClass() throws Exception {
147     TEST_UTIL.shutdownMiniCluster();
148   }
149 
150 
151   private static int getHConnectionManagerCacheSize(){
152     return HConnectionTestingUtility.getConnectionCount();
153   }
154 
155   @Test
156   public void testClusterConnection() throws IOException {
157     ThreadPoolExecutor otherPool = new ThreadPoolExecutor(1, 1,
158         5, TimeUnit.SECONDS,
159         new SynchronousQueue<Runnable>(),
160         Threads.newDaemonThreadFactory("test-hcm"));
161 
162     HConnection con1 = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
163     HConnection con2 = HConnectionManager.createConnection(TEST_UTIL.getConfiguration(), otherPool);
164     // make sure the internally created ExecutorService is the one passed
165     assertTrue(otherPool == ((HConnectionImplementation)con2).getCurrentBatchPool());
166 
167     String tableName = "testClusterConnection";
168     TEST_UTIL.createTable(tableName.getBytes(), FAM_NAM).close();
169     HTable t = (HTable)con1.getTable(tableName, otherPool);
170     // make sure passing a pool to the getTable does not trigger creation of an internal pool
171     assertNull("Internal Thread pool should be null", ((HConnectionImplementation)con1).getCurrentBatchPool());
172     // table should use the pool passed
173     assertTrue(otherPool == t.getPool());
174     t.close();
175 
176     t = (HTable)con2.getTable(tableName);
177     // table should use the connectin's internal pool
178     assertTrue(otherPool == t.getPool());
179     t.close();
180 
181     t = (HTable)con2.getTable(Bytes.toBytes(tableName));
182     // try other API too
183     assertTrue(otherPool == t.getPool());
184     t.close();
185 
186     t = (HTable)con2.getTable(TableName.valueOf(tableName));
187     // try other API too
188     assertTrue(otherPool == t.getPool());
189     t.close();
190 
191     t = (HTable)con1.getTable(tableName);
192     ExecutorService pool = ((HConnectionImplementation)con1).getCurrentBatchPool();
193     // make sure an internal pool was created
194     assertNotNull("An internal Thread pool should have been created", pool);
195     // and that the table is using it
196     assertTrue(t.getPool() == pool);
197     t.close();
198 
199     t = (HTable)con1.getTable(tableName);
200     // still using the *same* internal pool
201     assertTrue(t.getPool() == pool);
202     t.close();
203 
204     con1.close();
205     // if the pool was created on demand it should be closed upon connection close
206     assertTrue(pool.isShutdown());
207 
208     con2.close();
209     // if the pool is passed, it is not closed
210     assertFalse(otherPool.isShutdown());
211     otherPool.shutdownNow();
212   }
213 
214   /**
215    * Naive test to check that HConnection#getAdmin returns a properly constructed HBaseAdmin object
216    * @throws IOException Unable to construct admin
217    */
218   @Test
219   public void testAdminFactory() throws IOException {
220     Connection con1 = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
221     Admin admin = con1.getAdmin();
222     assertTrue(admin.getConnection() == con1);
223     assertTrue(admin.getConfiguration() == TEST_UTIL.getConfiguration());
224     con1.close();
225   }
226 
227   // Fails too often!  Needs work.  HBASE-12558
228   @Ignore @Test(expected = RegionServerStoppedException.class)
229   public void testClusterStatus() throws Exception {
230 
231     TableName tn =
232         TableName.valueOf("testClusterStatus");
233     byte[] cf = "cf".getBytes();
234     byte[] rk = "rk1".getBytes();
235 
236     JVMClusterUtil.RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
237     rs.waitForServerOnline();
238     final ServerName sn = rs.getRegionServer().getServerName();
239 
240     HTable t = TEST_UTIL.createTable(tn, cf);
241     TEST_UTIL.waitTableAvailable(tn);
242 
243     while(TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
244         getRegionStates().isRegionsInTransition()){
245       Thread.sleep(1);
246     }
247     final HConnectionImplementation hci =  (HConnectionImplementation)t.getConnection();
248     while (t.getRegionLocation(rk).getPort() != sn.getPort()){
249       TEST_UTIL.getHBaseAdmin().move(t.getRegionLocation(rk).getRegionInfo().
250           getEncodedNameAsBytes(), Bytes.toBytes(sn.toString()));
251       while(TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
252           getRegionStates().isRegionsInTransition()){
253         Thread.sleep(1);
254       }
255       hci.clearRegionCache(tn);
256     }
257     Assert.assertNotNull(hci.clusterStatusListener);
258     TEST_UTIL.assertRegionOnServer(t.getRegionLocation(rk).getRegionInfo(), sn, 20000);
259 
260     Put p1 = new Put(rk);
261     p1.add(cf, "qual".getBytes(), "val".getBytes());
262     t.put(p1);
263 
264     rs.getRegionServer().abort("I'm dead");
265 
266     // We want the status to be updated. That's a least 10 second
267     TEST_UTIL.waitFor(40000, 1000, true, new Waiter.Predicate<Exception>() {
268       @Override
269       public boolean evaluate() throws Exception {
270         return TEST_UTIL.getHBaseCluster().getMaster().getServerManager().
271             getDeadServers().isDeadServer(sn);
272       }
273     });
274 
275     TEST_UTIL.waitFor(40000, 1000, true, new Waiter.Predicate<Exception>() {
276       @Override
277       public boolean evaluate() throws Exception {
278         return hci.clusterStatusListener.isDeadServer(sn);
279       }
280     });
281 
282     t.close();
283     hci.getClient(sn);  // will throw an exception: RegionServerStoppedException
284   }
285 
286   /**
287    * Test that we can handle connection close: it will trigger a retry, but the calls will
288    *  finish.
289    */
290   @Test
291   public void testConnectionCloseAllowsInterrupt() throws Exception {
292     testConnectionClose(true);
293   }
294 
295   @Test
296   public void testConnectionNotAllowsInterrupt() throws Exception {
297     testConnectionClose(false);
298   }
299 
300   /**
301    * Test that an operation can fail if we read the global operation timeout, even if the
302    * individual timeout is fine. We do that with:
303    * - client side: an operation timeout of 30 seconds
304    * - server side: we sleep 20 second at each attempt. The first work fails, the second one
305    * succeeds. But the client won't wait that much, because 20 + 20 > 30, so the client
306    * timeouted when the server answers.
307    */
308   @Test
309   public void testOperationTimeout() throws Exception {
310     HTableDescriptor hdt = TEST_UTIL.createTableDescriptor("HCM-testOperationTimeout");
311     hdt.addCoprocessor(SleepAndFailFirstTime.class.getName());
312     HTable table = TEST_UTIL.createTable(hdt, new byte[][]{FAM_NAM}, TEST_UTIL.getConfiguration());
313     table.setRpcTimeout(Integer.MAX_VALUE);
314     // Check that it works if the timeout is big enough
315     table.setOperationTimeout(120 * 1000);
316     table.get(new Get(FAM_NAM));
317 
318     // Resetting and retrying. Will fail this time, not enough time for the second try
319     SleepAndFailFirstTime.ct.set(0);
320     try {
321       table.setOperationTimeout(30 * 1000);
322       table.get(new Get(FAM_NAM));
323       Assert.fail("We expect an exception here");
324     } catch (SocketTimeoutException e) {
325       // The client has a CallTimeout class, but it's not shared.We're not very clean today,
326       //  in the general case you can expect the call to stop, but the exception may vary.
327       // In this test however, we're sure that it will be a socket timeout.
328       LOG.info("We received an exception, as expected ", e);
329     } catch (IOException e) {
330       Assert.fail("Wrong exception:" + e.getMessage());
331     } finally {
332       table.close();
333     }
334   }
335 
336   @Test(expected = RetriesExhaustedException.class)
337   public void testRpcTimeout() throws Exception {
338     HTableDescriptor hdt = TEST_UTIL.createTableDescriptor("HCM-testRpcTimeout");
339     hdt.addCoprocessor(SleepCoprocessor.class.getName());
340     Configuration c = new Configuration(TEST_UTIL.getConfiguration());
341 
342     try (Table t = TEST_UTIL.createTable(hdt, new byte[][] { FAM_NAM }, c)) {
343       assert t instanceof HTable;
344       HTable table = (HTable) t;
345       table.setRpcTimeout(SleepCoprocessor.SLEEP_TIME / 2);
346       table.setOperationTimeout(SleepCoprocessor.SLEEP_TIME * 100);
347       table.get(new Get(FAM_NAM));
348     }
349   }
350 
351   private void testConnectionClose(boolean allowsInterrupt) throws Exception {
352     TableName tableName = TableName.valueOf("HCM-testConnectionClose" + allowsInterrupt);
353     TEST_UTIL.createTable(tableName, FAM_NAM).close();
354 
355     boolean previousBalance = TEST_UTIL.getHBaseAdmin().setBalancerRunning(false, true);
356 
357     Configuration c2 = new Configuration(TEST_UTIL.getConfiguration());
358     // We want to work on a separate connection.
359     c2.set(HConstants.HBASE_CLIENT_INSTANCE_ID, String.valueOf(-1));
360     c2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 100); // retry a lot
361     c2.setInt(HConstants.HBASE_CLIENT_PAUSE, 0); // don't wait between retries.
362     c2.setInt(RpcClient.FAILED_SERVER_EXPIRY_KEY, 0); // Server do not really expire
363     c2.setBoolean(RpcClient.SPECIFIC_WRITE_THREAD, allowsInterrupt);
364 
365     final HTable table = new HTable(c2, tableName);
366 
367     Put put = new Put(ROW);
368     put.add(FAM_NAM, ROW, ROW);
369     table.put(put);
370 
371     // 4 steps: ready=0; doGets=1; mustStop=2; stopped=3
372     final AtomicInteger step = new AtomicInteger(0);
373 
374     final AtomicReference<Throwable> failed = new AtomicReference<Throwable>(null);
375     Thread t = new Thread("testConnectionCloseThread") {
376       @Override
377       public void run() {
378         int done = 0;
379         try {
380           step.set(1);
381           while (step.get() == 1) {
382             Get get = new Get(ROW);
383             table.get(get);
384             done++;
385             if (done % 100 == 0)
386               LOG.info("done=" + done);
387           }
388         } catch (Throwable t) {
389           failed.set(t);
390           LOG.error(t);
391         }
392         step.set(3);
393       }
394     };
395     t.start();
396     TEST_UTIL.waitFor(20000, new Waiter.Predicate<Exception>() {
397       @Override
398       public boolean evaluate() throws Exception {
399         return step.get() == 1;
400       }
401     });
402 
403     ServerName sn = table.getRegionLocation(ROW).getServerName();
404     ConnectionManager.HConnectionImplementation conn =
405         (ConnectionManager.HConnectionImplementation) table.getConnection();
406     RpcClient rpcClient = conn.getRpcClient();
407 
408     LOG.info("Going to cancel connections. connection=" + conn.toString() + ", sn=" + sn);
409     for (int i = 0; i < 5000; i++) {
410       rpcClient.cancelConnections(sn);
411       Thread.sleep(5);
412     }
413 
414     step.compareAndSet(1, 2);
415     // The test may fail here if the thread doing the gets is stuck. The way to find
416     //  out what's happening is to look for the thread named 'testConnectionCloseThread'
417     TEST_UTIL.waitFor(40000, new Waiter.Predicate<Exception>() {
418       @Override
419       public boolean evaluate() throws Exception {
420         return step.get() == 3;
421       }
422     });
423     table.close();
424     Assert.assertTrue("Unexpected exception is " + failed.get(), failed.get() == null);
425     TEST_UTIL.getHBaseAdmin().setBalancerRunning(previousBalance, true);
426   }
427 
428   /**
429    * Test that connection can become idle without breaking everything.
430    */
431   @Test
432   public void testConnectionIdle() throws Exception {
433     TableName tableName = TableName.valueOf("HCM-testConnectionIdle");
434     TEST_UTIL.createTable(tableName, FAM_NAM).close();
435     int idleTime =  20000;
436     boolean previousBalance = TEST_UTIL.getHBaseAdmin().setBalancerRunning(false, true);
437 
438     Configuration c2 = new Configuration(TEST_UTIL.getConfiguration());
439     // We want to work on a separate connection.
440     c2.set(HConstants.HBASE_CLIENT_INSTANCE_ID, String.valueOf(-1));
441     c2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1); // Don't retry: retry = test failed
442     c2.setInt(RpcClient.IDLE_TIME, idleTime);
443 
444     final Table table = new HTable(c2, tableName);
445 
446     Put put = new Put(ROW);
447     put.add(FAM_NAM, ROW, ROW);
448     table.put(put);
449 
450     ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
451     mee.setValue(System.currentTimeMillis());
452     EnvironmentEdgeManager.injectEdge(mee);
453     LOG.info("first get");
454     table.get(new Get(ROW));
455 
456     LOG.info("first get - changing the time & sleeping");
457     mee.incValue(idleTime + 1000);
458     Thread.sleep(1500); // we need to wait a little for the connection to be seen as idle.
459                         // 1500 = sleep time in RpcClient#waitForWork + a margin
460 
461     LOG.info("second get - connection has been marked idle in the middle");
462     // To check that the connection actually became idle would need to read some private
463     //  fields of RpcClient.
464     table.get(new Get(ROW));
465     mee.incValue(idleTime + 1000);
466 
467     LOG.info("third get - connection is idle, but the reader doesn't know yet");
468     // We're testing here a special case:
469     //  time limit reached BUT connection not yet reclaimed AND a new call.
470     //  in this situation, we don't close the connection, instead we use it immediately.
471     // If we're very unlucky we can have a race condition in the test: the connection is already
472     //  under closing when we do the get, so we have an exception, and we don't retry as the
473     //  retry number is 1. The probability is very very low, and seems acceptable for now. It's
474     //  a test issue only.
475     table.get(new Get(ROW));
476 
477     LOG.info("we're done - time will change back");
478 
479     table.close();
480     EnvironmentEdgeManager.reset();
481     TEST_UTIL.getHBaseAdmin().setBalancerRunning(previousBalance, true);
482   }
483 
484     /**
485      * Test that the connection to the dead server is cut immediately when we receive the
486      *  notification.
487      * @throws Exception
488      */
489   @Test
490   public void testConnectionCut() throws Exception {
491 
492     TableName tableName = TableName.valueOf("HCM-testConnectionCut");
493 
494     TEST_UTIL.createTable(tableName, FAM_NAM).close();
495     boolean previousBalance = TEST_UTIL.getHBaseAdmin().setBalancerRunning(false, true);
496 
497     Configuration c2 = new Configuration(TEST_UTIL.getConfiguration());
498     // We want to work on a separate connection.
499     c2.set(HConstants.HBASE_CLIENT_INSTANCE_ID, String.valueOf(-1));
500     c2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
501     c2.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 30 * 1000);
502 
503     HTable table = new HTable(c2, tableName);
504 
505     Put p = new Put(FAM_NAM);
506     p.add(FAM_NAM, FAM_NAM, FAM_NAM);
507     table.put(p);
508 
509     final HConnectionImplementation hci =  (HConnectionImplementation)table.getConnection();
510     final HRegionLocation loc = table.getRegionLocation(FAM_NAM);
511 
512     Get get = new Get(FAM_NAM);
513     Assert.assertNotNull(table.get(get));
514 
515     get = new Get(FAM_NAM);
516     get.setFilter(new BlockingFilter());
517 
518     // This thread will mark the server as dead while we're waiting during a get.
519     Thread t = new Thread() {
520       @Override
521       public void run() {
522         synchronized (syncBlockingFilter) {
523           try {
524             syncBlockingFilter.wait();
525           } catch (InterruptedException e) {
526             throw new RuntimeException(e);
527           }
528         }
529         hci.clusterStatusListener.deadServerHandler.newDead(loc.getServerName());
530       }
531     };
532 
533     t.start();
534     try {
535       table.get(get);
536       Assert.fail();
537     } catch (IOException expected) {
538       LOG.debug("Received: " + expected);
539       Assert.assertFalse(expected instanceof SocketTimeoutException);
540       Assert.assertFalse(syncBlockingFilter.get());
541     } finally {
542       syncBlockingFilter.set(true);
543       t.join();
544       HConnectionManager.getConnection(c2).close();
545       TEST_UTIL.getHBaseAdmin().setBalancerRunning(previousBalance, true);
546     }
547 
548     table.close();
549   }
550 
551   protected static final AtomicBoolean syncBlockingFilter = new AtomicBoolean(false);
552 
553   public static class BlockingFilter extends FilterBase {
554     @Override
555     public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
556       int i = 0;
557       while (i++ < 1000 && !syncBlockingFilter.get()) {
558         synchronized (syncBlockingFilter) {
559           syncBlockingFilter.notifyAll();
560         }
561         Threads.sleep(100);
562       }
563       syncBlockingFilter.set(true);
564       return false;
565     }
566     @Override
567     public ReturnCode filterKeyValue(Cell ignored) throws IOException {
568       return ReturnCode.INCLUDE;
569     }
570 
571     public static Filter parseFrom(final byte [] pbBytes) throws DeserializationException{
572       return new BlockingFilter();
573     }
574   }
575 
576   @Test
577   public void abortingHConnectionRemovesItselfFromHCM() throws Exception {
578     // Save off current HConnections
579     Map<HConnectionKey, HConnectionImplementation> oldHBaseInstances =
580         new HashMap<HConnectionKey, HConnectionImplementation>();
581     oldHBaseInstances.putAll(ConnectionManager.CONNECTION_INSTANCES);
582 
583     ConnectionManager.CONNECTION_INSTANCES.clear();
584 
585     try {
586       HConnection connection = HConnectionManager.getConnection(TEST_UTIL.getConfiguration());
587       connection.abort("test abortingHConnectionRemovesItselfFromHCM", new Exception(
588           "test abortingHConnectionRemovesItselfFromHCM"));
589       Assert.assertNotSame(connection,
590         HConnectionManager.getConnection(TEST_UTIL.getConfiguration()));
591     } finally {
592       // Put original HConnections back
593       ConnectionManager.CONNECTION_INSTANCES.clear();
594       ConnectionManager.CONNECTION_INSTANCES.putAll(oldHBaseInstances);
595     }
596   }
597 
598   /**
599    * Test that when we delete a location using the first row of a region
600    * that we really delete it.
601    * @throws Exception
602    */
603   @Test
604   public void testRegionCaching() throws Exception{
605     TEST_UTIL.createMultiRegionTable(TABLE_NAME, FAM_NAM).close();
606     Configuration conf =  new Configuration(TEST_UTIL.getConfiguration());
607     conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
608     HTable table = new HTable(conf, TABLE_NAME);
609 
610     TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
611     Put put = new Put(ROW);
612     put.add(FAM_NAM, ROW, ROW);
613     table.put(put);
614     ConnectionManager.HConnectionImplementation conn =
615       (ConnectionManager.HConnectionImplementation)table.getConnection();
616 
617     assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
618 
619     final int nextPort = conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort() + 1;
620     HRegionLocation loc = conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation();
621     conn.updateCachedLocation(loc.getRegionInfo(), loc.getServerName(),
622         ServerName.valueOf("127.0.0.1", nextPort,
623         HConstants.LATEST_TIMESTAMP), HConstants.LATEST_TIMESTAMP);
624     Assert.assertEquals(conn.getCachedLocation(TABLE_NAME, ROW)
625       .getRegionLocation().getPort(), nextPort);
626 
627     conn.clearRegionCache(TABLE_NAME, ROW.clone());
628     RegionLocations rl = conn.getCachedLocation(TABLE_NAME, ROW);
629     assertNull("What is this location?? " + rl, rl);
630 
631     // We're now going to move the region and check that it works for the client
632     // First a new put to add the location in the cache
633     conn.clearRegionCache(TABLE_NAME);
634     Assert.assertEquals(0, conn.getNumberOfCachedRegionLocations(TABLE_NAME));
635     Put put2 = new Put(ROW);
636     put2.add(FAM_NAM, ROW, ROW);
637     table.put(put2);
638     assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
639     assertNotNull(conn.getCachedLocation(TableName.valueOf(TABLE_NAME.getName()), ROW.clone()));
640 
641     TEST_UTIL.getHBaseAdmin().setBalancerRunning(false, false);
642     HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
643 
644     // We can wait for all regions to be online, that makes log reading easier when debugging
645     while (master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
646       Thread.sleep(1);
647     }
648 
649     // Now moving the region to the second server
650     HRegionLocation toMove = conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation();
651     byte[] regionName = toMove.getRegionInfo().getRegionName();
652     byte[] encodedRegionNameBytes = toMove.getRegionInfo().getEncodedNameAsBytes();
653 
654     // Choose the other server.
655     int curServerId = TEST_UTIL.getHBaseCluster().getServerWith(regionName);
656     int destServerId = (curServerId == 0 ? 1 : 0);
657 
658     HRegionServer curServer = TEST_UTIL.getHBaseCluster().getRegionServer(curServerId);
659     HRegionServer destServer = TEST_UTIL.getHBaseCluster().getRegionServer(destServerId);
660 
661     ServerName destServerName = destServer.getServerName();
662 
663     // Check that we are in the expected state
664     Assert.assertTrue(curServer != destServer);
665     Assert.assertFalse(curServer.getServerName().equals(destServer.getServerName()));
666     Assert.assertFalse( toMove.getPort() == destServerName.getPort());
667     Assert.assertNotNull(curServer.getOnlineRegion(regionName));
668     Assert.assertNull(destServer.getOnlineRegion(regionName));
669     Assert.assertFalse(TEST_UTIL.getMiniHBaseCluster().getMaster().
670         getAssignmentManager().getRegionStates().isRegionsInTransition());
671 
672     // Moving. It's possible that we don't have all the regions online at this point, so
673     //  the test must depends only on the region we're looking at.
674     LOG.info("Move starting region="+toMove.getRegionInfo().getRegionNameAsString());
675     TEST_UTIL.getHBaseAdmin().move(
676       toMove.getRegionInfo().getEncodedNameAsBytes(),
677       destServerName.getServerName().getBytes()
678     );
679 
680     while (destServer.getOnlineRegion(regionName) == null ||
681         destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) ||
682         curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) ||
683         master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
684       // wait for the move to be finished
685       Thread.sleep(1);
686     }
687 
688     LOG.info("Move finished for region="+toMove.getRegionInfo().getRegionNameAsString());
689 
690     // Check our new state.
691     Assert.assertNull(curServer.getOnlineRegion(regionName));
692     Assert.assertNotNull(destServer.getOnlineRegion(regionName));
693     Assert.assertFalse(destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
694     Assert.assertFalse(curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
695 
696 
697     // Cache was NOT updated and points to the wrong server
698     Assert.assertFalse(
699         conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation()
700           .getPort() == destServerName.getPort());
701 
702     // This part relies on a number of tries equals to 1.
703     // We do a put and expect the cache to be updated, even if we don't retry
704     LOG.info("Put starting");
705     Put put3 = new Put(ROW);
706     put3.add(FAM_NAM, ROW, ROW);
707     try {
708       table.put(put3);
709       Assert.fail("Unreachable point");
710     } catch (RetriesExhaustedWithDetailsException e){
711       LOG.info("Put done, exception caught: " + e.getClass());
712       Assert.assertEquals(1, e.getNumExceptions());
713       Assert.assertEquals(1, e.getCauses().size());
714       Assert.assertArrayEquals(e.getRow(0).getRow(), ROW);
715 
716       // Check that we unserialized the exception as expected
717       Throwable cause = ClientExceptionsUtil.findException(e.getCause(0));
718       Assert.assertNotNull(cause);
719       Assert.assertTrue(cause instanceof RegionMovedException);
720     }
721     Assert.assertNotNull("Cached connection is null", conn.getCachedLocation(TABLE_NAME, ROW));
722     Assert.assertEquals(
723         "Previous server was " + curServer.getServerName().getHostAndPort(),
724         destServerName.getPort(),
725         conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort());
726 
727     Assert.assertFalse(destServer.getRegionsInTransitionInRS()
728       .containsKey(encodedRegionNameBytes));
729     Assert.assertFalse(curServer.getRegionsInTransitionInRS()
730       .containsKey(encodedRegionNameBytes));
731 
732     // We move it back to do another test with a scan
733     LOG.info("Move starting region=" + toMove.getRegionInfo().getRegionNameAsString());
734     TEST_UTIL.getHBaseAdmin().move(
735       toMove.getRegionInfo().getEncodedNameAsBytes(),
736       curServer.getServerName().getServerName().getBytes()
737     );
738 
739     while (curServer.getOnlineRegion(regionName) == null ||
740         destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) ||
741         curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) ||
742         master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
743       // wait for the move to be finished
744       Thread.sleep(1);
745     }
746 
747     // Check our new state.
748     Assert.assertNotNull(curServer.getOnlineRegion(regionName));
749     Assert.assertNull(destServer.getOnlineRegion(regionName));
750     LOG.info("Move finished for region=" + toMove.getRegionInfo().getRegionNameAsString());
751 
752     // Cache was NOT updated and points to the wrong server
753     Assert.assertFalse(conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort() ==
754       curServer.getServerName().getPort());
755 
756     Scan sc = new Scan();
757     sc.setStopRow(ROW);
758     sc.setStartRow(ROW);
759 
760     // The scanner takes the max retries from the connection configuration, not the table as
761     // the put.
762     TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
763 
764     try {
765       ResultScanner rs = table.getScanner(sc);
766       while (rs.next() != null) {
767       }
768       Assert.fail("Unreachable point");
769     } catch (RetriesExhaustedException e) {
770       LOG.info("Scan done, expected exception caught: " + e.getClass());
771     }
772 
773     // Cache is updated with the right value.
774     Assert.assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
775     Assert.assertEquals(
776       "Previous server was "+destServer.getServerName().getHostAndPort(),
777       curServer.getServerName().getPort(),
778       conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort());
779 
780     TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
781         HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
782     table.close();
783   }
784 
785   /**
786    * Test that Connection or Pool are not closed when managed externally
787    * @throws Exception
788    */
789   @Test
790   public void testConnectionManagement() throws Exception{
791     Table table0 = TEST_UTIL.createTable(TABLE_NAME1, FAM_NAM);
792     Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
793     HTable table = (HTable) conn.getTable(TABLE_NAME1);
794     table.close();
795     assertFalse(conn.isClosed());
796     assertFalse(table.getPool().isShutdown());
797     table = (HTable) conn.getTable(TABLE_NAME1);
798     table.close();
799     assertFalse(table.getPool().isShutdown());
800     conn.close();
801     assertTrue(table.getPool().isShutdown());
802     table0.close();
803   }
804 
805   /**
806    * Test that stale cache updates don't override newer cached values.
807    */
808   @Test(timeout = 60000)
809   public void testCacheSeqNums() throws Exception{
810     HTable table = TEST_UTIL.createMultiRegionTable(TABLE_NAME2, FAM_NAM);
811     Put put = new Put(ROW);
812     put.add(FAM_NAM, ROW, ROW);
813     table.put(put);
814     ConnectionManager.HConnectionImplementation conn =
815       (ConnectionManager.HConnectionImplementation)table.getConnection();
816 
817     HRegionLocation location = conn.getCachedLocation(TABLE_NAME2, ROW).getRegionLocation();
818     assertNotNull(location);
819 
820     ServerName anySource = ServerName.valueOf(location.getHostname(), location.getPort() - 1, 0L);
821 
822     // Same server as already in cache reporting - overwrites any value despite seqNum.
823     int nextPort = location.getPort() + 1;
824     conn.updateCachedLocation(location.getRegionInfo(), location.getServerName(),
825         ServerName.valueOf("127.0.0.1", nextPort, 0), location.getSeqNum() - 1);
826     location = conn.getCachedLocation(TABLE_NAME2, ROW).getRegionLocation();
827     Assert.assertEquals(nextPort, location.getPort());
828 
829     // No source specified - same.
830     nextPort = location.getPort() + 1;
831     conn.updateCachedLocation(location.getRegionInfo(), location.getServerName(),
832         ServerName.valueOf("127.0.0.1", nextPort, 0), location.getSeqNum() - 1);
833     location = conn.getCachedLocation(TABLE_NAME2, ROW).getRegionLocation();
834     Assert.assertEquals(nextPort, location.getPort());
835 
836     // Higher seqNum - overwrites lower seqNum.
837     nextPort = location.getPort() + 1;
838     conn.updateCachedLocation(location.getRegionInfo(), anySource,
839         ServerName.valueOf("127.0.0.1", nextPort, 0), location.getSeqNum() + 1);
840     location = conn.getCachedLocation(TABLE_NAME2, ROW).getRegionLocation();
841     Assert.assertEquals(nextPort, location.getPort());
842 
843     // Lower seqNum - does not overwrite higher seqNum.
844     nextPort = location.getPort() + 1;
845     conn.updateCachedLocation(location.getRegionInfo(), anySource,
846         ServerName.valueOf("127.0.0.1", nextPort, 0), location.getSeqNum() - 1);
847     location = conn.getCachedLocation(TABLE_NAME2, ROW).getRegionLocation();
848     Assert.assertEquals(nextPort - 1, location.getPort());
849     table.close();
850   }
851 
852   /**
853    * Make sure that {@link Configuration} instances that are essentially the
854    * same map to the same {@link HConnection} instance.
855    */
856   @Test
857   public void testConnectionSameness() throws Exception {
858     Connection previousConnection = null;
859     for (int i = 0; i < 2; i++) {
860       // set random key to differentiate the connection from previous ones
861       Configuration configuration = TEST_UTIL.getConfiguration();
862       configuration.set("some_key", String.valueOf(_randy.nextInt()));
863       LOG.info("The hash code of the current configuration is: "
864           + configuration.hashCode());
865       Connection currentConnection = HConnectionManager
866           .getConnection(configuration);
867       if (previousConnection != null) {
868         assertTrue(
869             "Did not get the same connection even though its key didn't change",
870             previousConnection == currentConnection);
871       }
872       previousConnection = currentConnection;
873       // change the configuration, so that it is no longer reachable from the
874       // client's perspective. However, since its part of the LRU doubly linked
875       // list, it will eventually get thrown out, at which time it should also
876       // close the corresponding {@link HConnection}.
877       configuration.set("other_key", String.valueOf(_randy.nextInt()));
878     }
879   }
880 
881   /**
882    * Makes sure that there is no leaking of
883    * {@link ConnectionManager.HConnectionImplementation} in the {@link HConnectionManager}
884    * class.
885    * @deprecated Tests deprecated functionality.  Remove in 1.0.
886    */
887   @Deprecated
888   @Test
889   public void testConnectionUniqueness() throws Exception {
890     int zkmaxconnections = TEST_UTIL.getConfiguration().
891       getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS,
892           HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS);
893     // Test up to a max that is < the maximum number of zk connections.  If we
894     // go above zk connections, we just fall into cycle where we are failing
895     // to set up a session and test runs for a long time.
896     int maxConnections = Math.min(zkmaxconnections - 1, 20);
897     List<HConnection> connections = new ArrayList<HConnection>(maxConnections);
898     Connection previousConnection = null;
899     try {
900       for (int i = 0; i < maxConnections; i++) {
901         // set random key to differentiate the connection from previous ones
902         Configuration configuration = new Configuration(TEST_UTIL.getConfiguration());
903         configuration.set("some_key", String.valueOf(_randy.nextInt()));
904         configuration.set(HConstants.HBASE_CLIENT_INSTANCE_ID,
905             String.valueOf(_randy.nextInt()));
906         LOG.info("The hash code of the current configuration is: "
907             + configuration.hashCode());
908         HConnection currentConnection =
909           HConnectionManager.getConnection(configuration);
910         if (previousConnection != null) {
911           assertTrue("Got the same connection even though its key changed!",
912               previousConnection != currentConnection);
913         }
914         // change the configuration, so that it is no longer reachable from the
915         // client's perspective. However, since its part of the LRU doubly linked
916         // list, it will eventually get thrown out, at which time it should also
917         // close the corresponding {@link HConnection}.
918         configuration.set("other_key", String.valueOf(_randy.nextInt()));
919 
920         previousConnection = currentConnection;
921         LOG.info("The current HConnectionManager#HBASE_INSTANCES cache size is: "
922             + getHConnectionManagerCacheSize());
923         Thread.sleep(50);
924         connections.add(currentConnection);
925       }
926     } finally {
927       for (Connection c: connections) {
928         // Clean up connections made so we don't interfere w/ subsequent tests.
929         HConnectionManager.deleteConnection(c.getConfiguration());
930       }
931     }
932   }
933 
934   @Test
935   public void testClosing() throws Exception {
936     Configuration configuration =
937       new Configuration(TEST_UTIL.getConfiguration());
938     configuration.set(HConstants.HBASE_CLIENT_INSTANCE_ID,
939         String.valueOf(_randy.nextInt()));
940 
941     Connection c1 = ConnectionFactory.createConnection(configuration);
942     // We create two connections with the same key.
943     Connection c2 = ConnectionFactory.createConnection(configuration);
944 
945     Connection c3 = HConnectionManager.getConnection(configuration);
946     Connection c4 = HConnectionManager.getConnection(configuration);
947     assertTrue(c3 == c4);
948 
949     c1.close();
950     assertTrue(c1.isClosed());
951     assertFalse(c2.isClosed());
952     assertFalse(c3.isClosed());
953 
954     c3.close();
955     // still a reference left
956     assertFalse(c3.isClosed());
957     c3.close();
958     assertTrue(c3.isClosed());
959     // c3 was removed from the cache
960     Connection c5 = HConnectionManager.getConnection(configuration);
961     assertTrue(c5 != c3);
962 
963     assertFalse(c2.isClosed());
964     c2.close();
965     assertTrue(c2.isClosed());
966     c5.close();
967     assertTrue(c5.isClosed());
968   }
969 
970   /**
971    * Trivial test to verify that nobody messes with
972    * {@link HConnectionManager#createConnection(Configuration)}
973    */
974   @Test
975   public void testCreateConnection() throws Exception {
976     Configuration configuration = TEST_UTIL.getConfiguration();
977     Connection c1 = ConnectionFactory.createConnection(configuration);
978     Connection c2 = ConnectionFactory.createConnection(configuration);
979     // created from the same configuration, yet they are different
980     assertTrue(c1 != c2);
981     assertTrue(c1.getConfiguration() == c2.getConfiguration());
982     // make sure these were not cached
983     Connection c3 = HConnectionManager.getConnection(configuration);
984     assertTrue(c1 != c3);
985     assertTrue(c2 != c3);
986   }
987 
988 
989   /**
990    * This test checks that one can connect to the cluster with only the
991    *  ZooKeeper quorum set. Other stuff like master address will be read
992    *  from ZK by the client.
993    */
994   @Test(timeout = 60000)
995   public void testConnection() throws Exception{
996     // We create an empty config and add the ZK address.
997     Configuration c = new Configuration();
998     c.set(HConstants.ZOOKEEPER_QUORUM,
999       TEST_UTIL.getConfiguration().get(HConstants.ZOOKEEPER_QUORUM));
1000     c.set(HConstants.ZOOKEEPER_CLIENT_PORT ,
1001       TEST_UTIL.getConfiguration().get(HConstants.ZOOKEEPER_CLIENT_PORT));
1002 
1003     // This should be enough to connect
1004     HConnection conn = HConnectionManager.getConnection(c);
1005     assertTrue( conn.isMasterRunning() );
1006     conn.close();
1007   }
1008 
1009   private int setNumTries(HConnectionImplementation hci, int newVal) throws Exception {
1010     Field numTries = hci.getClass().getDeclaredField("numTries");
1011     numTries.setAccessible(true);
1012     Field modifiersField = Field.class.getDeclaredField("modifiers");
1013     modifiersField.setAccessible(true);
1014     modifiersField.setInt(numTries, numTries.getModifiers() & ~Modifier.FINAL);
1015     final int prevNumRetriesVal = (Integer)numTries.get(hci);
1016     numTries.set(hci, newVal);
1017 
1018     return prevNumRetriesVal;
1019   }
1020 
1021   @Test (timeout=30000)
1022   public void testMulti() throws Exception {
1023     HTable table = TEST_UTIL.createMultiRegionTable(TABLE_NAME3, FAM_NAM);
1024      try {
1025        ConnectionManager.HConnectionImplementation conn =
1026            ( ConnectionManager.HConnectionImplementation)table.getConnection();
1027 
1028        // We're now going to move the region and check that it works for the client
1029        // First a new put to add the location in the cache
1030        conn.clearRegionCache(TABLE_NAME3);
1031        Assert.assertEquals(0, conn.getNumberOfCachedRegionLocations(TABLE_NAME3));
1032 
1033        TEST_UTIL.getHBaseAdmin().setBalancerRunning(false, false);
1034        HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
1035 
1036        // We can wait for all regions to be online, that makes log reading easier when debugging
1037        while (master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
1038          Thread.sleep(1);
1039        }
1040 
1041        Put put = new Put(ROW_X);
1042        put.add(FAM_NAM, ROW_X, ROW_X);
1043        table.put(put);
1044 
1045        // Now moving the region to the second server
1046        HRegionLocation toMove = conn.getCachedLocation(TABLE_NAME3, ROW_X).getRegionLocation();
1047        byte[] regionName = toMove.getRegionInfo().getRegionName();
1048        byte[] encodedRegionNameBytes = toMove.getRegionInfo().getEncodedNameAsBytes();
1049 
1050        // Choose the other server.
1051        int curServerId = TEST_UTIL.getHBaseCluster().getServerWith(regionName);
1052        int destServerId = (curServerId == 0 ? 1 : 0);
1053 
1054        HRegionServer curServer = TEST_UTIL.getHBaseCluster().getRegionServer(curServerId);
1055        HRegionServer destServer = TEST_UTIL.getHBaseCluster().getRegionServer(destServerId);
1056 
1057        ServerName destServerName = destServer.getServerName();
1058 
1059        //find another row in the cur server that is less than ROW_X
1060        List<Region> regions = curServer.getOnlineRegions(TABLE_NAME3);
1061        byte[] otherRow = null;
1062        for (Region region : regions) {
1063          if (!region.getRegionInfo().getEncodedName().equals(toMove.getRegionInfo().getEncodedName())
1064              && Bytes.BYTES_COMPARATOR.compare(region.getRegionInfo().getStartKey(), ROW_X) < 0) {
1065            otherRow = region.getRegionInfo().getStartKey();
1066            break;
1067          }
1068        }
1069        assertNotNull(otherRow);
1070        // If empty row, set it to first row.-f
1071        if (otherRow.length <= 0) otherRow = Bytes.toBytes("aaa");
1072        Put put2 = new Put(otherRow);
1073        put2.add(FAM_NAM, otherRow, otherRow);
1074        table.put(put2); //cache put2's location
1075 
1076        // Check that we are in the expected state
1077        Assert.assertTrue(curServer != destServer);
1078        Assert.assertNotEquals(curServer.getServerName(), destServer.getServerName());
1079        Assert.assertNotEquals(toMove.getPort(), destServerName.getPort());
1080        Assert.assertNotNull(curServer.getOnlineRegion(regionName));
1081        Assert.assertNull(destServer.getOnlineRegion(regionName));
1082        Assert.assertFalse(TEST_UTIL.getMiniHBaseCluster().getMaster().
1083            getAssignmentManager().getRegionStates().isRegionsInTransition());
1084 
1085        // Moving. It's possible that we don't have all the regions online at this point, so
1086        //  the test must depends only on the region we're looking at.
1087        LOG.info("Move starting region="+toMove.getRegionInfo().getRegionNameAsString());
1088        TEST_UTIL.getHBaseAdmin().move(
1089            toMove.getRegionInfo().getEncodedNameAsBytes(),
1090            destServerName.getServerName().getBytes()
1091            );
1092 
1093        while (destServer.getOnlineRegion(regionName) == null ||
1094            destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) ||
1095            curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) ||
1096            master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
1097          // wait for the move to be finished
1098          Thread.sleep(1);
1099         }
1100 
1101        LOG.info("Move finished for region="+toMove.getRegionInfo().getRegionNameAsString());
1102 
1103        // Check our new state.
1104        Assert.assertNull(curServer.getOnlineRegion(regionName));
1105        Assert.assertNotNull(destServer.getOnlineRegion(regionName));
1106        Assert.assertFalse(destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
1107        Assert.assertFalse(curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
1108 
1109 
1110        // Cache was NOT updated and points to the wrong server
1111        Assert.assertFalse(
1112            conn.getCachedLocation(TABLE_NAME3, ROW_X).getRegionLocation()
1113             .getPort() == destServerName.getPort());
1114 
1115        // Hijack the number of retry to fail after 2 tries
1116        final int prevNumRetriesVal = setNumTries(conn, 2);
1117 
1118        Put put3 = new Put(ROW_X);
1119        put3.add(FAM_NAM, ROW_X, ROW_X);
1120        Put put4 = new Put(otherRow);
1121        put4.add(FAM_NAM, otherRow, otherRow);
1122 
1123        // do multi
1124        table.batch(Lists.newArrayList(put4, put3)); // first should be a valid row,
1125        // second we get RegionMovedException.
1126 
1127        setNumTries(conn, prevNumRetriesVal);
1128      } finally {
1129        table.close();
1130      }
1131   }
1132 
1133   @Ignore ("Test presumes RETRY_BACKOFF will never change; it has") @Test
1134   public void testErrorBackoffTimeCalculation() throws Exception {
1135     // TODO: This test would seem to presume hardcoded RETRY_BACKOFF which it should not.
1136     final long ANY_PAUSE = 100;
1137     ServerName location = ServerName.valueOf("127.0.0.1", 1, 0);
1138     ServerName diffLocation = ServerName.valueOf("127.0.0.1", 2, 0);
1139 
1140     ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge();
1141     EnvironmentEdgeManager.injectEdge(timeMachine);
1142     try {
1143       long timeBase = timeMachine.currentTime();
1144       long largeAmountOfTime = ANY_PAUSE * 1000;
1145       ConnectionManager.ServerErrorTracker tracker =
1146           new ConnectionManager.ServerErrorTracker(largeAmountOfTime, 100);
1147 
1148       // The default backoff is 0.
1149       assertEquals(0, tracker.calculateBackoffTime(location, ANY_PAUSE));
1150 
1151       // Check some backoff values from HConstants sequence.
1152       tracker.reportServerError(location);
1153       assertEqualsWithJitter(ANY_PAUSE, tracker.calculateBackoffTime(location, ANY_PAUSE));
1154       tracker.reportServerError(location);
1155       tracker.reportServerError(location);
1156       tracker.reportServerError(location);
1157       assertEqualsWithJitter(ANY_PAUSE * 5, tracker.calculateBackoffTime(location, ANY_PAUSE));
1158 
1159       // All of this shouldn't affect backoff for different location.
1160       assertEquals(0, tracker.calculateBackoffTime(diffLocation, ANY_PAUSE));
1161       tracker.reportServerError(diffLocation);
1162       assertEqualsWithJitter(ANY_PAUSE, tracker.calculateBackoffTime(diffLocation, ANY_PAUSE));
1163 
1164       // Check with different base.
1165       assertEqualsWithJitter(ANY_PAUSE * 10,
1166           tracker.calculateBackoffTime(location, ANY_PAUSE * 2));
1167 
1168       // See that time from last error is taken into account. Time shift is applied after jitter,
1169       // so pass the original expected backoff as the base for jitter.
1170       long timeShift = (long)(ANY_PAUSE * 0.5);
1171       timeMachine.setValue(timeBase + timeShift);
1172       assertEqualsWithJitter((ANY_PAUSE * 5) - timeShift,
1173         tracker.calculateBackoffTime(location, ANY_PAUSE), ANY_PAUSE * 2);
1174 
1175       // However we should not go into negative.
1176       timeMachine.setValue(timeBase + ANY_PAUSE * 100);
1177       assertEquals(0, tracker.calculateBackoffTime(location, ANY_PAUSE));
1178 
1179       // We also should not go over the boundary; last retry would be on it.
1180       long timeLeft = (long)(ANY_PAUSE * 0.5);
1181       timeMachine.setValue(timeBase + largeAmountOfTime - timeLeft);
1182       assertTrue(tracker.canRetryMore(1));
1183       tracker.reportServerError(location);
1184       assertEquals(timeLeft, tracker.calculateBackoffTime(location, ANY_PAUSE));
1185       timeMachine.setValue(timeBase + largeAmountOfTime);
1186       assertFalse(tracker.canRetryMore(1));
1187     } finally {
1188       EnvironmentEdgeManager.reset();
1189     }
1190   }
1191 
1192   private static void assertEqualsWithJitter(long expected, long actual) {
1193     assertEqualsWithJitter(expected, actual, expected);
1194   }
1195 
1196   private static void assertEqualsWithJitter(long expected, long actual, long jitterBase) {
1197     assertTrue("Value not within jitter: " + expected + " vs " + actual,
1198         Math.abs(actual - expected) <= (0.01f * jitterBase));
1199   }
1200 
1201   /**
1202    * Tests that a destroyed connection does not have a live zookeeper.
1203    * Below is timing based.  We put up a connection to a table and then close the connection while
1204    * having a background thread running that is forcing close of the connection to try and
1205    * provoke a close catastrophe; we are hoping for a car crash so we can see if we are leaking
1206    * zk connections.
1207    * @throws Exception
1208    */
1209   @Ignore ("Flakey test: See HBASE-8996")@Test
1210   public void testDeleteForZKConnLeak() throws Exception {
1211     TEST_UTIL.createTable(TABLE_NAME4, FAM_NAM);
1212     final Configuration config = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
1213     config.setInt("zookeeper.recovery.retry", 1);
1214     config.setInt("zookeeper.recovery.retry.intervalmill", 1000);
1215     config.setInt("hbase.rpc.timeout", 2000);
1216     config.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
1217 
1218     ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 10,
1219       5, TimeUnit.SECONDS,
1220       new SynchronousQueue<Runnable>(),
1221       Threads.newDaemonThreadFactory("test-hcm-delete"));
1222 
1223     pool.submit(new Runnable() {
1224       @Override
1225       public void run() {
1226         while (!Thread.interrupted()) {
1227           try {
1228             HConnection conn = HConnectionManager.getConnection(config);
1229             LOG.info("Connection " + conn);
1230             HConnectionManager.deleteStaleConnection(conn);
1231             LOG.info("Connection closed " + conn);
1232             // TODO: This sleep time should be less than the time that it takes to open and close
1233             // a table.  Ideally we would do a few runs first to measure.  For now this is
1234             // timing based; hopefully we hit the bad condition.
1235             Threads.sleep(10);
1236           } catch (Exception e) {
1237           }
1238         }
1239       }
1240     });
1241 
1242     // Use connection multiple times.
1243     for (int i = 0; i < 30; i++) {
1244       Connection c1 = null;
1245       try {
1246         c1 = ConnectionManager.getConnectionInternal(config);
1247         LOG.info("HTable connection " + i + " " + c1);
1248         Table table = new HTable(config, TABLE_NAME4, pool);
1249         table.close();
1250         LOG.info("HTable connection " + i + " closed " + c1);
1251       } catch (Exception e) {
1252         LOG.info("We actually want this to happen!!!!  So we can see if we are leaking zk", e);
1253       } finally {
1254         if (c1 != null) {
1255           if (c1.isClosed()) {
1256             // cannot use getZooKeeper as method instantiates watcher if null
1257             Field zkwField = c1.getClass().getDeclaredField("keepAliveZookeeper");
1258             zkwField.setAccessible(true);
1259             Object watcher = zkwField.get(c1);
1260 
1261             if (watcher != null) {
1262               if (((ZooKeeperWatcher)watcher).getRecoverableZooKeeper().getState().isAlive()) {
1263                 // non-synchronized access to watcher; sleep and check again in case zk connection
1264                 // hasn't been cleaned up yet.
1265                 Thread.sleep(1000);
1266                 if (((ZooKeeperWatcher) watcher).getRecoverableZooKeeper().getState().isAlive()) {
1267                   pool.shutdownNow();
1268                   fail("Live zookeeper in closed connection");
1269                 }
1270               }
1271             }
1272           }
1273           c1.close();
1274         }
1275       }
1276     }
1277     pool.shutdownNow();
1278   }
1279 
1280   @Test(timeout = 60000)
1281   public void testConnectionRideOverClusterRestart() throws IOException, InterruptedException {
1282     Configuration config = new Configuration(TEST_UTIL.getConfiguration());
1283 
1284     TableName tableName = TableName.valueOf("testConnectionRideOverClusterRestart");
1285     TEST_UTIL.createTable(tableName.getName(), new byte[][] {FAM_NAM}, config).close();
1286 
1287     Connection connection = ConnectionFactory.createConnection(config);
1288     Table table = connection.getTable(tableName);
1289 
1290     // this will cache the meta location and table's region location
1291     table.get(new Get(Bytes.toBytes("foo")));
1292 
1293     // restart HBase
1294     TEST_UTIL.shutdownMiniHBaseCluster();
1295     TEST_UTIL.restartHBaseCluster(2);
1296     // this should be able to discover new locations for meta and table's region
1297     table.get(new Get(Bytes.toBytes("foo")));
1298     TEST_UTIL.deleteTable(tableName);
1299     table.close();
1300     connection.close();
1301   }
1302 }
1303