001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.List;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtility;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.client.TableDescriptor;
032import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
033import org.apache.hadoop.hbase.io.HFileLink;
034import org.apache.hadoop.hbase.monitoring.MonitoredTask;
035import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
036import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
037import org.apache.hadoop.hbase.testclassification.RegionServerTests;
038import org.apache.hadoop.hbase.testclassification.SmallTests;
039import org.apache.hadoop.hbase.util.FSTableDescriptors;
040import org.apache.hadoop.hbase.util.FSUtils;
041import org.junit.After;
042import org.junit.Before;
043import org.junit.ClassRule;
044import org.junit.Test;
045import org.junit.experimental.categories.Category;
046import org.mockito.Mockito;
047import org.slf4j.Logger;
048import org.slf4j.LoggerFactory;
049
050import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
051
052/**
053 * Test the restore/clone operation from a file-system point of view.
054 */
055@Category({RegionServerTests.class, SmallTests.class})
056public class TestRestoreSnapshotHelper {
057
058  @ClassRule
059  public static final HBaseClassTestRule CLASS_RULE =
060      HBaseClassTestRule.forClass(TestRestoreSnapshotHelper.class);
061
062  private static final Logger LOG = LoggerFactory.getLogger(TestRestoreSnapshotHelper.class);
063
064  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
065  protected final static String TEST_HFILE = "abc";
066
067  protected Configuration conf;
068  protected Path archiveDir;
069  protected FileSystem fs;
070  protected Path rootDir;
071
072  protected void setupConf(Configuration conf) {
073  }
074
075  @Before
076  public void setup() throws Exception {
077    rootDir = TEST_UTIL.getDataTestDir("testRestore");
078    archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
079    fs = TEST_UTIL.getTestFileSystem();
080    conf = TEST_UTIL.getConfiguration();
081    setupConf(conf);
082    FSUtils.setRootDir(conf, rootDir);
083  }
084
085  @After
086  public void tearDown() throws Exception {
087    fs.delete(TEST_UTIL.getDataTestDir(), true);
088  }
089
090  protected SnapshotMock createSnapshotMock() throws IOException {
091    return new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
092  }
093
094  @Test
095  public void testRestore() throws IOException {
096    restoreAndVerify("snapshot", "testRestore");
097  }
098
099  @Test
100  public void testRestoreWithNamespace() throws IOException {
101    restoreAndVerify("snapshot", "namespace1:testRestoreWithNamespace");
102  }
103
104  private void restoreAndVerify(final String snapshotName, final String tableName) throws IOException {
105    // Test Rolling-Upgrade like Snapshot.
106    // half machines writing using v1 and the others using v2 format.
107    SnapshotMock snapshotMock = createSnapshotMock();
108    SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("snapshot", tableName);
109    builder.addRegionV1();
110    builder.addRegionV2();
111    builder.addRegionV2();
112    builder.addRegionV1();
113    Path snapshotDir = builder.commit();
114    TableDescriptor htd = builder.getTableDescriptor();
115    SnapshotDescription desc = builder.getSnapshotDescription();
116
117    // Test clone a snapshot
118    TableDescriptor htdClone = snapshotMock.createHtd("testtb-clone");
119    testRestore(snapshotDir, desc, htdClone);
120    verifyRestore(rootDir, htd, htdClone);
121
122    // Test clone a clone ("link to link")
123    SnapshotDescription cloneDesc = SnapshotDescription.newBuilder()
124        .setName("cloneSnapshot")
125        .setTable("testtb-clone")
126        .build();
127    Path cloneDir = FSUtils.getTableDir(rootDir, htdClone.getTableName());
128    TableDescriptor htdClone2 = snapshotMock.createHtd("testtb-clone2");
129    testRestore(cloneDir, cloneDesc, htdClone2);
130    verifyRestore(rootDir, htd, htdClone2);
131  }
132
133  private void verifyRestore(final Path rootDir, final TableDescriptor sourceHtd,
134      final TableDescriptor htdClone) throws IOException {
135    List<String> files = SnapshotTestingUtils.listHFileNames(fs,
136      FSUtils.getTableDir(rootDir, htdClone.getTableName()));
137    assertEquals(12, files.size());
138    for (int i = 0; i < files.size(); i += 2) {
139      String linkFile = files.get(i);
140      String refFile = files.get(i+1);
141      assertTrue(linkFile + " should be a HFileLink", HFileLink.isHFileLink(linkFile));
142      assertTrue(refFile + " should be a Referene", StoreFileInfo.isReference(refFile));
143      assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(linkFile));
144      Path refPath = getReferredToFile(refFile);
145      LOG.debug("get reference name for file " + refFile + " = " + refPath);
146      assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName()));
147      assertEquals(linkFile, refPath.getName());
148    }
149  }
150
151  /**
152   * Execute the restore operation
153   * @param snapshotDir The snapshot directory to use as "restore source"
154   * @param sd The snapshot descriptor
155   * @param htdClone The HTableDescriptor of the table to restore/clone.
156   */
157  private void testRestore(final Path snapshotDir, final SnapshotDescription sd,
158      final TableDescriptor htdClone) throws IOException {
159    LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
160    FSUtils.logFileSystemState(fs, rootDir, LOG);
161
162    new FSTableDescriptors(conf).createTableDescriptor(htdClone);
163    RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sd, htdClone);
164    helper.restoreHdfsRegions();
165
166    LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
167    FSUtils.logFileSystemState(fs, rootDir, LOG);
168  }
169
170  /**
171   * Initialize the restore helper, based on the snapshot and table information provided.
172   */
173  private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir,
174      final SnapshotDescription sd, final TableDescriptor htdClone) throws IOException {
175    ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
176    MonitoredTask status = Mockito.mock(MonitoredTask.class);
177
178    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
179    return new RestoreSnapshotHelper(conf, fs, manifest,
180      htdClone, rootDir, monitor, status);
181  }
182
183  private Path getReferredToFile(final String referenceName) {
184    Path fakeBasePath = new Path(new Path("table", "region"), "cf");
185    return StoreFileInfo.getReferredToFile(new Path(fakeBasePath, referenceName));
186  }
187}