View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.replication;
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.hadoop.conf.Configuration;
24  import org.apache.hadoop.hbase.HBaseConfiguration;
25  import org.apache.hadoop.hbase.HBaseTestingUtility;
26  import org.apache.hadoop.hbase.HColumnDescriptor;
27  import org.apache.hadoop.hbase.HConstants;
28  import org.apache.hadoop.hbase.HTableDescriptor;
29  import org.apache.hadoop.hbase.TableName;
30  import org.apache.hadoop.hbase.client.Admin;
31  import org.apache.hadoop.hbase.client.Connection;
32  import org.apache.hadoop.hbase.client.ConnectionFactory;
33  import org.apache.hadoop.hbase.client.Table;
34  import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
35  import org.apache.hadoop.hbase.util.Bytes;
36  import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
37  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
38  import org.junit.AfterClass;
39  import org.junit.BeforeClass;
40  
41  /**
42   * This class is only a base for other integration-level replication tests.
43   * Do not add tests here.
44   * TestReplicationSmallTests is where tests that don't require bring machines up/down should go
45   * All other tests should have their own classes and extend this one
46   */
47  public class TestReplicationBase {
48  /*
49    {
50      ((Log4JLogger) ReplicationSource.LOG).getLogger().setLevel(Level.ALL);
51    }*/
52  
53    private static final Log LOG = LogFactory.getLog(TestReplicationBase.class);
54  
55    protected static Configuration conf1 = HBaseConfiguration.create();
56    protected static Configuration conf2;
57    protected static Configuration CONF_WITH_LOCALFS;
58  
59    protected static ZooKeeperWatcher zkw1;
60    protected static ZooKeeperWatcher zkw2;
61  
62    protected static ReplicationAdmin admin;
63  
64    protected static Table htable1;
65    protected static Table htable2;
66  
67    protected static HBaseTestingUtility utility1;
68    protected static HBaseTestingUtility utility2;
69    protected static final int NB_ROWS_IN_BATCH = 100;
70    protected static final int NB_ROWS_IN_BIG_BATCH =
71        NB_ROWS_IN_BATCH * 10;
72    protected static final long SLEEP_TIME = 500;
73    protected static final int NB_RETRIES = 10;
74  
75    protected static final TableName tableName = TableName.valueOf("test");
76    protected static final byte[] famName = Bytes.toBytes("f");
77    protected static final byte[] row = Bytes.toBytes("row");
78    protected static final byte[] noRepfamName = Bytes.toBytes("norep");
79  
80    /**
81     * @throws java.lang.Exception
82     */
83    @BeforeClass
84    public static void setUpBeforeClass() throws Exception {
85      conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
86      // We don't want too many edits per batch sent to the ReplicationEndpoint to trigger
87      // sufficient number of events. But we don't want to go too low because
88      // HBaseInterClusterReplicationEndpoint partitions entries into batches and we want
89      // more than one batch sent to the peer cluster for better testing.
90      conf1.setInt("replication.source.size.capacity", 102400);
91      conf1.setLong("replication.source.sleepforretries", 100);
92      conf1.setInt("hbase.regionserver.maxlogs", 10);
93      conf1.setLong("hbase.master.logcleaner.ttl", 10);
94      conf1.setInt("zookeeper.recovery.retry", 1);
95      conf1.setInt("zookeeper.recovery.retry.intervalmill", 10);
96      conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
97      conf1.setBoolean("dfs.support.append", true);
98      conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
99      conf1.setInt("replication.stats.thread.period.seconds", 5);
100     conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);
101     conf1.setLong("replication.sleep.before.failover", 2000);
102     conf1.setInt("replication.source.maxretriesmultiplier", 10);
103     conf1.setFloat("replication.source.ratio", 1.0f);
104     conf1.setBoolean("replication.source.eof.autorecovery", true);
105 
106     utility1 = new HBaseTestingUtility(conf1);
107     utility1.startMiniZKCluster();
108     MiniZooKeeperCluster miniZK = utility1.getZkCluster();
109     // Have to reget conf1 in case zk cluster location different
110     // than default
111     conf1 = utility1.getConfiguration();  
112     zkw1 = new ZooKeeperWatcher(conf1, "cluster1", null, true);
113     admin = new ReplicationAdmin(conf1);
114     LOG.info("Setup first Zk");
115 
116     // Base conf2 on conf1 so it gets the right zk cluster.
117     conf2 = HBaseConfiguration.create(conf1);
118     conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
119     conf2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
120     conf2.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
121     conf2.setBoolean("dfs.support.append", true);
122     conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false);
123 
124     utility2 = new HBaseTestingUtility(conf2);
125     utility2.setZkCluster(miniZK);
126     zkw2 = new ZooKeeperWatcher(conf2, "cluster2", null, true);
127 
128     admin.addPeer("2", utility2.getClusterKey());
129 
130     LOG.info("Setup second Zk");
131     CONF_WITH_LOCALFS = HBaseConfiguration.create(conf1);
132     utility1.startMiniCluster(2);
133     // Have a bunch of slave servers, because inter-cluster shipping logic uses number of sinks
134     // as a component in deciding maximum number of parallel batches to send to the peer cluster.
135     utility2.startMiniCluster(4);
136 
137     HTableDescriptor table = new HTableDescriptor(tableName);
138     HColumnDescriptor fam = new HColumnDescriptor(famName);
139     fam.setMaxVersions(100);
140     fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
141     table.addFamily(fam);
142     fam = new HColumnDescriptor(noRepfamName);
143     table.addFamily(fam);
144     Connection connection1 = ConnectionFactory.createConnection(conf1);
145     Connection connection2 = ConnectionFactory.createConnection(conf2);
146     try (Admin admin1 = connection1.getAdmin()) {
147       admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
148     }
149     try (Admin admin2 = connection2.getAdmin()) {
150       admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
151     }
152     utility1.waitUntilAllRegionsAssigned(tableName);
153     utility2.waitUntilAllRegionsAssigned(tableName);
154     htable1 = connection1.getTable(tableName);
155     htable1.setWriteBufferSize(1024);
156     htable2 = connection2.getTable(tableName);
157   }
158 
159   /**
160    * @throws java.lang.Exception
161    */
162   @AfterClass
163   public static void tearDownAfterClass() throws Exception {
164     htable2.close();
165     htable1.close();
166     admin.close();
167     utility2.shutdownMiniCluster();
168     utility1.shutdownMiniCluster();
169   }
170 }