Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.fail;
023
024import java.io.IOException;
025import java.util.HashSet;
026import java.util.List;
027import java.util.Set;
028
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.Path;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtility;
033import org.apache.hadoop.hbase.HColumnDescriptor;
034import org.apache.hadoop.hbase.HConstants;
035import org.apache.hadoop.hbase.HTableDescriptor;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.master.MasterFileSystem;
038import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
039import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
040import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
041import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
042import org.apache.hadoop.hbase.testclassification.ClientTests;
043import org.apache.hadoop.hbase.testclassification.LargeTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.apache.hadoop.hbase.util.FSUtils;
046import org.junit.After;
047import org.junit.AfterClass;
048import org.junit.Before;
049import org.junit.BeforeClass;
050import org.junit.ClassRule;
051import org.junit.Rule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.junit.rules.TestName;
055
056/**
057 * Test restore snapshots from the client
058 */
059@Category({LargeTests.class, ClientTests.class})
060public class TestRestoreSnapshotFromClient {
061
062  @ClassRule
063  public static final HBaseClassTestRule CLASS_RULE =
064      HBaseClassTestRule.forClass(TestRestoreSnapshotFromClient.class);
065
066  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
067
068  protected final byte[] FAMILY = Bytes.toBytes("cf");
069  protected final byte[] TEST_FAMILY2 = Bytes.toBytes("cf2");
070
071  protected TableName tableName;
072  protected byte[] emptySnapshot;
073  protected byte[] snapshotName0;
074  protected byte[] snapshotName1;
075  protected byte[] snapshotName2;
076  protected int snapshot0Rows;
077  protected int snapshot1Rows;
078  protected Admin admin;
079
080  @Rule
081  public TestName name = new TestName();
082
083  @BeforeClass
084  public static void setupCluster() throws Exception {
085    setupConf(TEST_UTIL.getConfiguration());
086    TEST_UTIL.startMiniCluster(3);
087  }
088
089  protected static void setupConf(Configuration conf) {
090    TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
091    TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
092    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
093    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
094    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
095    TEST_UTIL.getConfiguration().setBoolean(
096        "hbase.master.enabletable.roundrobin", true);
097  }
098
099  @AfterClass
100  public static void tearDownAfterClass() throws Exception {
101    TEST_UTIL.shutdownMiniCluster();
102  }
103
104  /**
105   * Initialize the tests with a table filled with some data
106   * and two snapshots (snapshotName0, snapshotName1) of different states.
107   * The tableName, snapshotNames and the number of rows in the snapshot are initialized.
108   */
109  @Before
110  public void setup() throws Exception {
111    this.admin = TEST_UTIL.getAdmin();
112
113    long tid = System.currentTimeMillis();
114    tableName = TableName.valueOf(name.getMethodName() + "-" + tid);
115    emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid);
116    snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
117    snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
118    snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
119
120    // create Table and disable it
121    createTable();
122    admin.disableTable(tableName);
123
124    // take an empty snapshot
125    admin.snapshot(emptySnapshot, tableName);
126
127    // enable table and insert data
128    admin.enableTable(tableName);
129    SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
130    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
131      snapshot0Rows = countRows(table);
132    }
133    admin.disableTable(tableName);
134
135    // take a snapshot
136    admin.snapshot(snapshotName0, tableName);
137
138    // enable table and insert more data
139    admin.enableTable(tableName);
140    SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
141    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
142      snapshot1Rows = countRows(table);
143    }
144  }
145
146  protected void createTable() throws Exception {
147    SnapshotTestingUtils.createTable(TEST_UTIL, tableName, getNumReplicas(), FAMILY);
148  }
149
150  @After
151  public void tearDown() throws Exception {
152    TEST_UTIL.deleteTable(tableName);
153    SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getAdmin());
154    SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
155  }
156
157  @Test
158  public void testRestoreSnapshot() throws IOException {
159    verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
160    admin.disableTable(tableName);
161    admin.snapshot(snapshotName1, tableName);
162    // Restore from snapshot-0
163    admin.restoreSnapshot(snapshotName0);
164    admin.enableTable(tableName);
165    verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
166    SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
167
168    // Restore from emptySnapshot
169    admin.disableTable(tableName);
170    admin.restoreSnapshot(emptySnapshot);
171    admin.enableTable(tableName);
172    verifyRowCount(TEST_UTIL, tableName, 0);
173    SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
174
175    // Restore from snapshot-1
176    admin.disableTable(tableName);
177    admin.restoreSnapshot(snapshotName1);
178    admin.enableTable(tableName);
179    verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
180    SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
181
182    // Restore from snapshot-1
183    TEST_UTIL.deleteTable(tableName);
184    admin.restoreSnapshot(snapshotName1);
185    verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
186    SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
187  }
188
189  protected int getNumReplicas() {
190    return 1;
191  }
192
193  protected HColumnDescriptor getTestRestoreSchemaChangeHCD() {
194    return new HColumnDescriptor(TEST_FAMILY2);
195  }
196
197  @Test
198  public void testRestoreSchemaChange() throws Exception {
199    Table table = TEST_UTIL.getConnection().getTable(tableName);
200
201    // Add one column family and put some data in it
202    admin.disableTable(tableName);
203    admin.addColumnFamily(tableName, getTestRestoreSchemaChangeHCD());
204    admin.enableTable(tableName);
205    assertEquals(2, table.getTableDescriptor().getFamilies().size());
206    HTableDescriptor htd = admin.getTableDescriptor(tableName);
207    assertEquals(2, htd.getFamilies().size());
208    SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, TEST_FAMILY2);
209    long snapshot2Rows = snapshot1Rows + 500L;
210    assertEquals(snapshot2Rows, countRows(table));
211    assertEquals(500, countRows(table, TEST_FAMILY2));
212    Set<String> fsFamilies = getFamiliesFromFS(tableName);
213    assertEquals(2, fsFamilies.size());
214
215    // Take a snapshot
216    admin.disableTable(tableName);
217    admin.snapshot(snapshotName2, tableName);
218
219    // Restore the snapshot (without the cf)
220    admin.restoreSnapshot(snapshotName0);
221    admin.enableTable(tableName);
222    assertEquals(1, table.getTableDescriptor().getFamilies().size());
223    try {
224      countRows(table, TEST_FAMILY2);
225      fail("family '" + Bytes.toString(TEST_FAMILY2) + "' should not exists");
226    } catch (NoSuchColumnFamilyException e) {
227      // expected
228    }
229    assertEquals(snapshot0Rows, countRows(table));
230    htd = admin.getTableDescriptor(tableName);
231    assertEquals(1, htd.getFamilies().size());
232    fsFamilies = getFamiliesFromFS(tableName);
233    assertEquals(1, fsFamilies.size());
234
235    // Restore back the snapshot (with the cf)
236    admin.disableTable(tableName);
237    admin.restoreSnapshot(snapshotName2);
238    admin.enableTable(tableName);
239    htd = admin.getTableDescriptor(tableName);
240    assertEquals(2, htd.getFamilies().size());
241    assertEquals(2, table.getTableDescriptor().getFamilies().size());
242    assertEquals(500, countRows(table, TEST_FAMILY2));
243    assertEquals(snapshot2Rows, countRows(table));
244    fsFamilies = getFamiliesFromFS(tableName);
245    assertEquals(2, fsFamilies.size());
246    table.close();
247  }
248
249  @Test
250  public void testCloneSnapshotOfCloned() throws IOException, InterruptedException {
251    TableName clonedTableName = TableName.valueOf(name.getMethodName() + "-" + System.currentTimeMillis());
252    admin.cloneSnapshot(snapshotName0, clonedTableName);
253    verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
254    SnapshotTestingUtils.verifyReplicasCameOnline(clonedTableName, admin, getNumReplicas());
255    admin.disableTable(clonedTableName);
256    admin.snapshot(snapshotName2, clonedTableName);
257    TEST_UTIL.deleteTable(clonedTableName);
258    waitCleanerRun();
259
260    admin.cloneSnapshot(snapshotName2, clonedTableName);
261    verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
262    SnapshotTestingUtils.verifyReplicasCameOnline(clonedTableName, admin, getNumReplicas());
263    TEST_UTIL.deleteTable(clonedTableName);
264  }
265
266  @Test
267  public void testCloneAndRestoreSnapshot() throws IOException, InterruptedException {
268    TEST_UTIL.deleteTable(tableName);
269    waitCleanerRun();
270
271    admin.cloneSnapshot(snapshotName0, tableName);
272    verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
273    SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
274    waitCleanerRun();
275
276    admin.disableTable(tableName);
277    admin.restoreSnapshot(snapshotName0);
278    admin.enableTable(tableName);
279    verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
280    SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
281  }
282
283  @Test
284  public void testCorruptedSnapshot() throws IOException, InterruptedException {
285    SnapshotTestingUtils.corruptSnapshot(TEST_UTIL, Bytes.toString(snapshotName0));
286    TableName cloneName = TableName.valueOf(name.getMethodName() + "-" + System.currentTimeMillis());
287    try {
288      admin.cloneSnapshot(snapshotName0, cloneName);
289      fail("Expected CorruptedSnapshotException, got succeeded cloneSnapshot()");
290    } catch (CorruptedSnapshotException e) {
291      // Got the expected corruption exception.
292      // check for no references of the cloned table.
293      assertFalse(admin.tableExists(cloneName));
294    } catch (Exception e) {
295      fail("Expected CorruptedSnapshotException got: " + e);
296    }
297  }
298
299  @Test
300  public void testRestoreSnapshotAfterSplittingRegions() throws IOException, InterruptedException {
301    List<RegionInfo> regionInfos = admin.getRegions(tableName);
302    RegionReplicaUtil.removeNonDefaultRegions(regionInfos);
303
304    // Split the first region
305    splitRegion(regionInfos.get(0));
306
307    // Take a snapshot
308    admin.snapshot(snapshotName1, tableName);
309
310    // Restore the snapshot
311    admin.disableTable(tableName);
312    admin.restoreSnapshot(snapshotName1);
313    admin.enableTable(tableName);
314
315    verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
316  }
317
318  // ==========================================================================
319  //  Helpers
320  // ==========================================================================
321  private void waitCleanerRun() throws InterruptedException {
322    TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
323  }
324
325  private Set<String> getFamiliesFromFS(final TableName tableName) throws IOException {
326    MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
327    Set<String> families = new HashSet<>();
328    Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
329    for (Path regionDir: FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) {
330      for (Path familyDir: FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) {
331        families.add(familyDir.getName());
332      }
333    }
334    return families;
335  }
336
337  protected void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
338      long expectedRows) throws IOException {
339    SnapshotTestingUtils.verifyRowCount(util, tableName, expectedRows);
340  }
341
342  protected int countRows(final Table table, final byte[]... families) throws IOException {
343    return TEST_UTIL.countRows(table, families);
344  }
345
346  protected void splitRegion(final RegionInfo regionInfo) throws IOException {
347    byte[][] splitPoints = Bytes.split(regionInfo.getStartKey(), regionInfo.getEndKey(), 1);
348    admin.split(regionInfo.getTable(), splitPoints[1]);
349  }
350}