View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.replication;
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.hadoop.conf.Configuration;
24  import org.apache.hadoop.hbase.HBaseConfiguration;
25  import org.apache.hadoop.hbase.HBaseTestingUtility;
26  import org.apache.hadoop.hbase.HColumnDescriptor;
27  import org.apache.hadoop.hbase.HConstants;
28  import org.apache.hadoop.hbase.HTableDescriptor;
29  import org.apache.hadoop.hbase.TableName;
30  import org.apache.hadoop.hbase.client.Admin;
31  import org.apache.hadoop.hbase.client.Connection;
32  import org.apache.hadoop.hbase.client.ConnectionFactory;
33  import org.apache.hadoop.hbase.client.Table;
34  import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
35  import org.apache.hadoop.hbase.util.Bytes;
36  import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
37  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
38  import org.junit.AfterClass;
39  import org.junit.BeforeClass;
40  
41  /**
42   * This class is only a base for other integration-level replication tests.
43   * Do not add tests here.
44   * TestReplicationSmallTests is where tests that don't require bring machines up/down should go
45   * All other tests should have their own classes and extend this one
46   */
47  public class TestReplicationBase {
48  /*
49    {
50      ((Log4JLogger) ReplicationSource.LOG).getLogger().setLevel(Level.ALL);
51    }*/
52  
53    private static final Log LOG = LogFactory.getLog(TestReplicationBase.class);
54  
55    protected static Configuration conf1 = HBaseConfiguration.create();
56    protected static Configuration conf2;
57    protected static Configuration CONF_WITH_LOCALFS;
58  
59    protected static ZooKeeperWatcher zkw1;
60    protected static ZooKeeperWatcher zkw2;
61  
62    protected static ReplicationAdmin admin;
63  
64    protected static Table htable1;
65    protected static Table htable2;
66  
67    protected static HBaseTestingUtility utility1;
68    protected static HBaseTestingUtility utility2;
69    protected static final String PEER_ID = "2";
70    protected static final int NB_ROWS_IN_BATCH = 100;
71    protected static final int NB_ROWS_IN_BIG_BATCH =
72        NB_ROWS_IN_BATCH * 10;
73    protected static final long SLEEP_TIME = 500;
74    protected static final int NB_RETRIES = 10;
75  
76    protected static final TableName tableName = TableName.valueOf("test");
77    protected static final byte[] famName = Bytes.toBytes("f");
78    protected static final byte[] row = Bytes.toBytes("row");
79    protected static final byte[] noRepfamName = Bytes.toBytes("norep");
80  
81    /**
82     * @throws java.lang.Exception
83     */
84    @BeforeClass
85    public static void setUpBeforeClass() throws Exception {
86      conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
87      // We don't want too many edits per batch sent to the ReplicationEndpoint to trigger
88      // sufficient number of events. But we don't want to go too low because
89      // HBaseInterClusterReplicationEndpoint partitions entries into batches and we want
90      // more than one batch sent to the peer cluster for better testing.
91      conf1.setInt("replication.source.size.capacity", 102400);
92      conf1.setLong("replication.source.sleepforretries", 100);
93      conf1.setInt("hbase.regionserver.maxlogs", 10);
94      conf1.setLong("hbase.master.logcleaner.ttl", 10);
95      conf1.setInt("zookeeper.recovery.retry", 1);
96      conf1.setInt("zookeeper.recovery.retry.intervalmill", 10);
97      conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
98      conf1.setBoolean("dfs.support.append", true);
99      conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
100     conf1.setInt("replication.stats.thread.period.seconds", 5);
101     conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);
102     conf1.setLong("replication.sleep.before.failover", 2000);
103     conf1.setInt("replication.source.maxretriesmultiplier", 10);
104     conf1.setFloat("replication.source.ratio", 1.0f);
105     conf1.setBoolean("replication.source.eof.autorecovery", true);
106 
107     utility1 = new HBaseTestingUtility(conf1);
108     utility1.startMiniZKCluster();
109     MiniZooKeeperCluster miniZK = utility1.getZkCluster();
110     // Have to reget conf1 in case zk cluster location different
111     // than default
112     conf1 = utility1.getConfiguration();  
113     zkw1 = new ZooKeeperWatcher(conf1, "cluster1", null, true);
114     admin = new ReplicationAdmin(conf1);
115     LOG.info("Setup first Zk");
116 
117     // Base conf2 on conf1 so it gets the right zk cluster.
118     conf2 = HBaseConfiguration.create(conf1);
119     conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
120     conf2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
121     conf2.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
122     conf2.setBoolean("dfs.support.append", true);
123     conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false);
124 
125     utility2 = new HBaseTestingUtility(conf2);
126     utility2.setZkCluster(miniZK);
127     zkw2 = new ZooKeeperWatcher(conf2, "cluster2", null, true);
128 
129     ReplicationPeerConfig rpc = new ReplicationPeerConfig();
130     rpc.setClusterKey(utility2.getClusterKey());
131     admin.addPeer(PEER_ID, rpc);
132 
133     LOG.info("Setup second Zk");
134     CONF_WITH_LOCALFS = HBaseConfiguration.create(conf1);
135     utility1.startMiniCluster(2);
136     // Have a bunch of slave servers, because inter-cluster shipping logic uses number of sinks
137     // as a component in deciding maximum number of parallel batches to send to the peer cluster.
138     utility2.startMiniCluster(4);
139 
140     HTableDescriptor table = new HTableDescriptor(tableName);
141     HColumnDescriptor fam = new HColumnDescriptor(famName);
142     fam.setMaxVersions(100);
143     fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
144     table.addFamily(fam);
145     fam = new HColumnDescriptor(noRepfamName);
146     table.addFamily(fam);
147     Connection connection1 = ConnectionFactory.createConnection(conf1);
148     Connection connection2 = ConnectionFactory.createConnection(conf2);
149     try (Admin admin1 = connection1.getAdmin()) {
150       admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
151     }
152     try (Admin admin2 = connection2.getAdmin()) {
153       admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
154     }
155     utility1.waitUntilAllRegionsAssigned(tableName);
156     utility2.waitUntilAllRegionsAssigned(tableName);
157     htable1 = connection1.getTable(tableName);
158     htable1.setWriteBufferSize(1024);
159     htable2 = connection2.getTable(tableName);
160   }
161 
162   /**
163    * @throws java.lang.Exception
164    */
165   @AfterClass
166   public static void tearDownAfterClass() throws Exception {
167     htable2.close();
168     htable1.close();
169     admin.close();
170     utility2.shutdownMiniCluster();
171     utility1.shutdownMiniCluster();
172   }
173 }