001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import java.io.IOException;
021import org.apache.hadoop.hbase.HBaseClassTestRule;
022import org.apache.hadoop.hbase.HBaseTestingUtility;
023import org.apache.hadoop.hbase.HConstants;
024import org.apache.hadoop.hbase.NamespaceDescriptor;
025import org.apache.hadoop.hbase.NamespaceNotFoundException;
026import org.apache.hadoop.hbase.TableName;
027import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
028import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
029import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
030import org.apache.hadoop.hbase.testclassification.ClientTests;
031import org.apache.hadoop.hbase.testclassification.LargeTests;
032import org.apache.hadoop.hbase.util.Bytes;
033import org.junit.After;
034import org.junit.AfterClass;
035import org.junit.Before;
036import org.junit.BeforeClass;
037import org.junit.ClassRule;
038import org.junit.Rule;
039import org.junit.Test;
040import org.junit.experimental.categories.Category;
041import org.junit.rules.TestName;
042import org.slf4j.Logger;
043import org.slf4j.LoggerFactory;
044
045/**
046 * Test clone snapshots from the client
047 */
048@Category({LargeTests.class, ClientTests.class})
049public class TestCloneSnapshotFromClient {
050
051  @ClassRule
052  public static final HBaseClassTestRule CLASS_RULE =
053      HBaseClassTestRule.forClass(TestCloneSnapshotFromClient.class);
054
055  private static final Logger LOG = LoggerFactory.getLogger(TestCloneSnapshotFromClient.class);
056
057  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
058
059  protected final byte[] FAMILY = Bytes.toBytes("cf");
060
061  protected byte[] emptySnapshot;
062  protected byte[] snapshotName0;
063  protected byte[] snapshotName1;
064  protected byte[] snapshotName2;
065  protected TableName tableName;
066  protected int snapshot0Rows;
067  protected int snapshot1Rows;
068  protected Admin admin;
069
070  @Rule
071  public TestName name = new TestName();
072
073  protected static void setupConfiguration() {
074    TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
075    TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
076    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
077    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
078    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
079    TEST_UTIL.getConfiguration().setBoolean(
080        "hbase.master.enabletable.roundrobin", true);
081  }
082
083  @BeforeClass
084  public static void setUpBeforeClass() throws Exception {
085    setupConfiguration();
086    TEST_UTIL.startMiniCluster(3);
087  }
088
089  @AfterClass
090  public static void tearDownAfterClass() throws Exception {
091    TEST_UTIL.shutdownMiniCluster();
092  }
093
094  /**
095   * Initialize the tests with a table filled with some data
096   * and two snapshots (snapshotName0, snapshotName1) of different states.
097   * The tableName, snapshotNames and the number of rows in the snapshot are initialized.
098   */
099  @Before
100  public void setup() throws Exception {
101    this.admin = TEST_UTIL.getAdmin();
102
103    long tid = System.currentTimeMillis();
104    tableName = TableName.valueOf(name.getMethodName() + tid);
105    emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid);
106    snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
107    snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
108    snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
109
110    createTableAndSnapshots();
111  }
112
113  protected void createTableAndSnapshots() throws Exception {
114    // create Table and disable it
115    SnapshotTestingUtils.createTable(TEST_UTIL, tableName, getNumReplicas(), FAMILY);
116    admin.disableTable(tableName);
117
118    // take an empty snapshot
119    admin.snapshot(emptySnapshot, tableName);
120
121    // enable table and insert data
122    admin.enableTable(tableName);
123    SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
124    try (Table table = TEST_UTIL.getConnection().getTable(tableName)){
125      snapshot0Rows = TEST_UTIL.countRows(table);
126    }
127    admin.disableTable(tableName);
128
129    // take a snapshot
130    admin.snapshot(snapshotName0, tableName);
131
132    // enable table and insert more data
133    admin.enableTable(tableName);
134    SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
135    try (Table table = TEST_UTIL.getConnection().getTable(tableName)){
136      snapshot1Rows = TEST_UTIL.countRows(table);
137    }
138    admin.disableTable(tableName);
139
140    // take a snapshot of the updated table
141    admin.snapshot(snapshotName1, tableName);
142
143    // re-enable table
144    admin.enableTable(tableName);
145  }
146
147  protected int getNumReplicas() {
148    return 1;
149  }
150
151  @After
152  public void tearDown() throws Exception {
153    if (admin.tableExists(tableName)) {
154      TEST_UTIL.deleteTable(tableName);
155    }
156    SnapshotTestingUtils.deleteAllSnapshots(admin);
157    SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
158  }
159
160  @Test(expected=SnapshotDoesNotExistException.class)
161  public void testCloneNonExistentSnapshot() throws IOException, InterruptedException {
162    String snapshotName = "random-snapshot-" + System.currentTimeMillis();
163    final TableName tableName = TableName.valueOf(name.getMethodName() + "-" + System.currentTimeMillis());
164    admin.cloneSnapshot(snapshotName, tableName);
165  }
166
167  @Test(expected = NamespaceNotFoundException.class)
168  public void testCloneOnMissingNamespace() throws IOException, InterruptedException {
169    final TableName clonedTableName = TableName.valueOf("unknownNS:" + name.getMethodName());
170    admin.cloneSnapshot(snapshotName1, clonedTableName);
171  }
172
173  @Test
174  public void testCloneSnapshot() throws IOException, InterruptedException {
175    final TableName clonedTableName = TableName.valueOf(name.getMethodName() + "-" + System.currentTimeMillis());
176    testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows);
177    testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows);
178    testCloneSnapshot(clonedTableName, emptySnapshot, 0);
179  }
180
181  private void testCloneSnapshot(final TableName tableName, final byte[] snapshotName,
182      int snapshotRows) throws IOException, InterruptedException {
183    // create a new table from snapshot
184    admin.cloneSnapshot(snapshotName, tableName);
185    verifyRowCount(TEST_UTIL, tableName, snapshotRows);
186
187    verifyReplicasCameOnline(tableName);
188    TEST_UTIL.deleteTable(tableName);
189  }
190
191  protected void verifyReplicasCameOnline(TableName tableName) throws IOException {
192    SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
193  }
194
195  @Test
196  public void testCloneSnapshotCrossNamespace() throws IOException, InterruptedException {
197    String nsName = "testCloneSnapshotCrossNamespace";
198    admin.createNamespace(NamespaceDescriptor.create(nsName).build());
199    final TableName clonedTableName = TableName.valueOf(nsName, name.getMethodName() + "-" + System.currentTimeMillis());
200    testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows);
201    testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows);
202    testCloneSnapshot(clonedTableName, emptySnapshot, 0);
203  }
204
205  /**
206   * Verify that tables created from the snapshot are still alive after source table deletion.
207   */
208  @Test
209  public void testCloneLinksAfterDelete() throws IOException, InterruptedException {
210    // Clone a table from the first snapshot
211    final TableName clonedTableName = TableName.valueOf(name.getMethodName() + "1-" + System.currentTimeMillis());
212    admin.cloneSnapshot(snapshotName0, clonedTableName);
213    verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
214
215    // Take a snapshot of this cloned table.
216    admin.disableTable(clonedTableName);
217    admin.snapshot(snapshotName2, clonedTableName);
218
219    // Clone the snapshot of the cloned table
220    final TableName clonedTableName2 = TableName.valueOf(name.getMethodName() + "2-" + System.currentTimeMillis());
221    admin.cloneSnapshot(snapshotName2, clonedTableName2);
222    verifyRowCount(TEST_UTIL, clonedTableName2, snapshot0Rows);
223    admin.disableTable(clonedTableName2);
224
225    // Remove the original table
226    TEST_UTIL.deleteTable(tableName);
227    waitCleanerRun();
228
229    // Verify the first cloned table
230    admin.enableTable(clonedTableName);
231    verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
232
233    // Verify the second cloned table
234    admin.enableTable(clonedTableName2);
235    verifyRowCount(TEST_UTIL, clonedTableName2, snapshot0Rows);
236    admin.disableTable(clonedTableName2);
237
238    // Delete the first cloned table
239    TEST_UTIL.deleteTable(clonedTableName);
240    waitCleanerRun();
241
242    // Verify the second cloned table
243    admin.enableTable(clonedTableName2);
244    verifyRowCount(TEST_UTIL, clonedTableName2, snapshot0Rows);
245
246    // Clone a new table from cloned
247    final TableName clonedTableName3 = TableName.valueOf(name.getMethodName() + "3-" + System.currentTimeMillis());
248    admin.cloneSnapshot(snapshotName2, clonedTableName3);
249    verifyRowCount(TEST_UTIL, clonedTableName3, snapshot0Rows);
250
251    // Delete the cloned tables
252    TEST_UTIL.deleteTable(clonedTableName2);
253    TEST_UTIL.deleteTable(clonedTableName3);
254    admin.deleteSnapshot(snapshotName2);
255  }
256
257  // ==========================================================================
258  //  Helpers
259  // ==========================================================================
260
261  private void waitCleanerRun() throws InterruptedException {
262    TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
263  }
264
265  protected void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
266      long expectedRows) throws IOException {
267    SnapshotTestingUtils.verifyRowCount(util, tableName, expectedRows);
268  }
269}