View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.io.OutputStream;
24  import java.util.ArrayList;
25  import java.util.Arrays;
26  import java.util.HashMap;
27  import java.util.HashSet;
28  import java.util.LinkedList;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.Map.Entry;
32  import java.util.Set;
33  import java.util.TreeMap;
34  import java.util.concurrent.ThreadPoolExecutor;
35  
36  import com.google.common.collect.ListMultimap;
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.hbase.classification.InterfaceAudience;
40  import org.apache.hadoop.conf.Configuration;
41  import org.apache.hadoop.fs.FileStatus;
42  import org.apache.hadoop.fs.FileSystem;
43  import org.apache.hadoop.fs.Path;
44  import org.apache.hadoop.hbase.HColumnDescriptor;
45  import org.apache.hadoop.hbase.HRegionInfo;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.TableName;
48  import org.apache.hadoop.hbase.backup.HFileArchiver;
49  import org.apache.hadoop.hbase.MetaTableAccessor;
50  import org.apache.hadoop.hbase.client.Connection;
51  import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
52  import org.apache.hadoop.hbase.io.HFileLink;
53  import org.apache.hadoop.hbase.io.Reference;
54  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
55  import org.apache.hadoop.hbase.mob.MobUtils;
56  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
57  import org.apache.hadoop.hbase.monitoring.TaskMonitor;
58  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
59  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
60  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
61  import org.apache.hadoop.hbase.regionserver.HRegion;
62  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
63  import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
64  import org.apache.hadoop.hbase.security.access.AccessControlClient;
65  import org.apache.hadoop.hbase.security.access.TablePermission;
66  import org.apache.hadoop.hbase.util.Bytes;
67  import org.apache.hadoop.hbase.util.FSUtils;
68  import org.apache.hadoop.hbase.util.ModifyRegionUtils;
69  import org.apache.hadoop.hbase.util.Pair;
70  import org.apache.hadoop.io.IOUtils;
71  
72  /**
73   * Helper to Restore/Clone a Snapshot
74   *
75   * <p>The helper assumes that a table is already created, and by calling restore()
76   * the content present in the snapshot will be restored as the new content of the table.
77   *
78   * <p>Clone from Snapshot: If the target table is empty, the restore operation
79   * is just a "clone operation", where the only operations are:
80   * <ul>
81   *  <li>for each region in the snapshot create a new region
82   *    (note that the region will have a different name, since the encoding contains the table name)
83   *  <li>for each file in the region create a new HFileLink to point to the original file.
84   *  <li>restore the logs, if any
85   * </ul>
86   *
87   * <p>Restore from Snapshot:
88   * <ul>
89   *  <li>for each region in the table verify which are available in the snapshot and which are not
90   *    <ul>
91   *    <li>if the region is not present in the snapshot, remove it.
92   *    <li>if the region is present in the snapshot
93   *      <ul>
94   *      <li>for each file in the table region verify which are available in the snapshot
95   *        <ul>
96   *          <li>if the hfile is not present in the snapshot, remove it
97   *          <li>if the hfile is present, keep it (nothing to do)
98   *        </ul>
99   *      <li>for each file in the snapshot region but not in the table
100  *        <ul>
101  *          <li>create a new HFileLink that point to the original file
102  *        </ul>
103  *      </ul>
104  *    </ul>
105  *  <li>for each region in the snapshot not present in the current table state
106  *    <ul>
107  *    <li>create a new region and for each file in the region create a new HFileLink
108  *      (This is the same as the clone operation)
109  *    </ul>
110  *  <li>restore the logs, if any
111  * </ul>
112  */
113 @InterfaceAudience.Private
114 public class RestoreSnapshotHelper {
115   private static final Log LOG = LogFactory.getLog(RestoreSnapshotHelper.class);
116 
117   private final Map<byte[], byte[]> regionsMap =
118         new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
119 
120   private final Map<String, Pair<String, String> > parentsMap =
121       new HashMap<String, Pair<String, String> >();
122 
123   private final ForeignExceptionDispatcher monitor;
124   private final MonitoredTask status;
125 
126   private final SnapshotManifest snapshotManifest;
127   private final SnapshotDescription snapshotDesc;
128   private final TableName snapshotTable;
129 
130   private final HTableDescriptor tableDesc;
131   private final Path rootDir;
132   private final Path tableDir;
133 
134   private final Configuration conf;
135   private final FileSystem fs;
136   private final boolean createBackRefs;
137 
138   public RestoreSnapshotHelper(final Configuration conf,
139       final FileSystem fs,
140       final SnapshotManifest manifest,
141       final HTableDescriptor tableDescriptor,
142       final Path rootDir,
143       final ForeignExceptionDispatcher monitor,
144       final MonitoredTask status) {
145     this(conf, fs, manifest, tableDescriptor, rootDir, monitor, status, true);
146   }
147 
148   public RestoreSnapshotHelper(final Configuration conf,
149       final FileSystem fs,
150       final SnapshotManifest manifest,
151       final HTableDescriptor tableDescriptor,
152       final Path rootDir,
153       final ForeignExceptionDispatcher monitor,
154       final MonitoredTask status,
155       final boolean createBackRefs)
156   {
157     this.fs = fs;
158     this.conf = conf;
159     this.snapshotManifest = manifest;
160     this.snapshotDesc = manifest.getSnapshotDescription();
161     this.snapshotTable = TableName.valueOf(snapshotDesc.getTable());
162     this.tableDesc = tableDescriptor;
163     this.rootDir = rootDir;
164     this.tableDir = FSUtils.getTableDir(rootDir, tableDesc.getTableName());
165     this.monitor = monitor;
166     this.status = status;
167     this.createBackRefs = createBackRefs;
168   }
169 
170   /**
171    * Restore the on-disk table to a specified snapshot state.
172    * @return the set of regions touched by the restore operation
173    */
174   public RestoreMetaChanges restoreHdfsRegions() throws IOException {
175     ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "RestoreSnapshot");
176     try {
177       return restoreHdfsRegions(exec);
178     } finally {
179       exec.shutdown();
180     }
181   }
182 
183   private RestoreMetaChanges restoreHdfsRegions(final ThreadPoolExecutor exec) throws IOException {
184     LOG.debug("starting restore");
185 
186     Map<String, SnapshotRegionManifest> regionManifests = snapshotManifest.getRegionManifestsMap();
187     if (regionManifests == null) {
188       LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
189       return null;
190     }
191 
192     RestoreMetaChanges metaChanges = new RestoreMetaChanges(parentsMap);
193 
194     // Take a copy of the manifest.keySet() since we are going to modify
195     // this instance, by removing the regions already present in the restore dir.
196     Set<String> regionNames = new HashSet<String>(regionManifests.keySet());
197 
198     HRegionInfo mobRegion = MobUtils.getMobRegionInfo(snapshotManifest.getTableDescriptor()
199         .getTableName());
200     // Identify which region are still available and which not.
201     // NOTE: we rely upon the region name as: "table name, start key, end key"
202     List<HRegionInfo> tableRegions = getTableRegions();
203     if (tableRegions != null) {
204       monitor.rethrowException();
205       for (HRegionInfo regionInfo: tableRegions) {
206         String regionName = regionInfo.getEncodedName();
207         if (regionNames.contains(regionName)) {
208           LOG.info("region to restore: " + regionName);
209           regionNames.remove(regionName);
210           metaChanges.addRegionToRestore(regionInfo);
211         } else {
212           LOG.info("region to remove: " + regionName);
213           metaChanges.addRegionToRemove(regionInfo);
214         }
215       }
216 
217       // Restore regions using the snapshot data
218       monitor.rethrowException();
219       status.setStatus("Restoring table regions...");
220       if (regionNames.contains(mobRegion.getEncodedName())) {
221         // restore the mob region in case
222         List<HRegionInfo> mobRegions = new ArrayList<HRegionInfo>(1);
223         mobRegions.add(mobRegion);
224         restoreHdfsMobRegions(exec, regionManifests, mobRegions);
225         regionNames.remove(mobRegion.getEncodedName());
226       }
227       restoreHdfsRegions(exec, regionManifests, metaChanges.getRegionsToRestore());
228       status.setStatus("Finished restoring all table regions.");
229 
230       // Remove regions from the current table
231       monitor.rethrowException();
232       status.setStatus("Starting to delete excess regions from table");
233       removeHdfsRegions(exec, metaChanges.getRegionsToRemove());
234       status.setStatus("Finished deleting excess regions from table.");
235     }
236 
237     // Regions to Add: present in the snapshot but not in the current table
238     if (regionNames.size() > 0) {
239       List<HRegionInfo> regionsToAdd = new ArrayList<HRegionInfo>(regionNames.size());
240 
241       monitor.rethrowException();
242       // add the mob region
243       if (regionNames.contains(mobRegion.getEncodedName())) {
244         cloneHdfsMobRegion(regionManifests, mobRegion);
245         regionNames.remove(mobRegion.getEncodedName());
246       }
247       for (String regionName: regionNames) {
248         LOG.info("region to add: " + regionName);
249         regionsToAdd.add(HRegionInfo.convert(regionManifests.get(regionName).getRegionInfo()));
250       }
251 
252       // Create new regions cloning from the snapshot
253       monitor.rethrowException();
254       status.setStatus("Cloning regions...");
255       HRegionInfo[] clonedRegions = cloneHdfsRegions(exec, regionManifests, regionsToAdd);
256       metaChanges.setNewRegions(clonedRegions);
257       status.setStatus("Finished cloning regions.");
258     }
259 
260     return metaChanges;
261   }
262 
263   /**
264    * Describe the set of operations needed to update hbase:meta after restore.
265    */
266   public static class RestoreMetaChanges {
267     private final Map<String, Pair<String, String> > parentsMap;
268 
269     private List<HRegionInfo> regionsToRestore = null;
270     private List<HRegionInfo> regionsToRemove = null;
271     private List<HRegionInfo> regionsToAdd = null;
272 
273     RestoreMetaChanges(final Map<String, Pair<String, String> > parentsMap) {
274       this.parentsMap = parentsMap;
275     }
276 
277     /**
278      * @return true if there're new regions
279      */
280     public boolean hasRegionsToAdd() {
281       return this.regionsToAdd != null && this.regionsToAdd.size() > 0;
282     }
283 
284     /**
285      * Returns the list of new regions added during the on-disk restore.
286      * The caller is responsible to add the regions to META.
287      * e.g MetaTableAccessor.addRegionsToMeta(...)
288      * @return the list of regions to add to META
289      */
290     public List<HRegionInfo> getRegionsToAdd() {
291       return this.regionsToAdd;
292     }
293 
294     /**
295      * @return true if there're regions to restore
296      */
297     public boolean hasRegionsToRestore() {
298       return this.regionsToRestore != null && this.regionsToRestore.size() > 0;
299     }
300 
301     /**
302      * Returns the list of 'restored regions' during the on-disk restore.
303      * The caller is responsible to add the regions to hbase:meta if not present.
304      * @return the list of regions restored
305      */
306     public List<HRegionInfo> getRegionsToRestore() {
307       return this.regionsToRestore;
308     }
309 
310     /**
311      * @return true if there're regions to remove
312      */
313     public boolean hasRegionsToRemove() {
314       return this.regionsToRemove != null && this.regionsToRemove.size() > 0;
315     }
316 
317     /**
318      * Returns the list of regions removed during the on-disk restore.
319      * The caller is responsible to remove the regions from META.
320      * e.g. MetaTableAccessor.deleteRegions(...)
321      * @return the list of regions to remove from META
322      */
323     public List<HRegionInfo> getRegionsToRemove() {
324       return this.regionsToRemove;
325     }
326 
327     void setNewRegions(final HRegionInfo[] hris) {
328       if (hris != null) {
329         regionsToAdd = Arrays.asList(hris);
330       } else {
331         regionsToAdd = null;
332       }
333     }
334 
335     void addRegionToRemove(final HRegionInfo hri) {
336       if (regionsToRemove == null) {
337         regionsToRemove = new LinkedList<HRegionInfo>();
338       }
339       regionsToRemove.add(hri);
340     }
341 
342     void addRegionToRestore(final HRegionInfo hri) {
343       if (regionsToRestore == null) {
344         regionsToRestore = new LinkedList<HRegionInfo>();
345       }
346       regionsToRestore.add(hri);
347     }
348 
349     public void updateMetaParentRegions(Connection connection,
350         final List<HRegionInfo> regionInfos) throws IOException {
351       if (regionInfos == null || parentsMap.isEmpty()) return;
352 
353       // Extract region names and offlined regions
354       Map<String, HRegionInfo> regionsByName = new HashMap<String, HRegionInfo>(regionInfos.size());
355       List<HRegionInfo> parentRegions = new LinkedList<>();
356       for (HRegionInfo regionInfo: regionInfos) {
357         if (regionInfo.isSplitParent()) {
358           parentRegions.add(regionInfo);
359         } else {
360           regionsByName.put(regionInfo.getEncodedName(), regionInfo);
361         }
362       }
363 
364       // Update Offline parents
365       for (HRegionInfo regionInfo: parentRegions) {
366         Pair<String, String> daughters = parentsMap.get(regionInfo.getEncodedName());
367         if (daughters == null) {
368           // The snapshot contains an unreferenced region.
369           // It will be removed by the CatalogJanitor.
370           LOG.warn("Skip update of unreferenced offline parent: " + regionInfo);
371           continue;
372         }
373 
374         // One side of the split is already compacted
375         if (daughters.getSecond() == null) {
376           daughters.setSecond(daughters.getFirst());
377         }
378 
379         LOG.debug("Update splits parent " + regionInfo.getEncodedName() + " -> " + daughters);
380         MetaTableAccessor.addRegionToMeta(connection, regionInfo,
381           regionsByName.get(daughters.getFirst()),
382           regionsByName.get(daughters.getSecond()));
383       }
384     }
385   }
386 
387   /**
388    * Remove specified regions from the file-system, using the archiver.
389    */
390   private void removeHdfsRegions(final ThreadPoolExecutor exec, final List<HRegionInfo> regions)
391       throws IOException {
392     if (regions == null || regions.size() == 0) return;
393     ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
394       @Override
395       public void editRegion(final HRegionInfo hri) throws IOException {
396         HFileArchiver.archiveRegion(conf, fs, hri);
397       }
398     });
399   }
400 
401   /**
402    * Restore specified regions by restoring content to the snapshot state.
403    */
404   private void restoreHdfsRegions(final ThreadPoolExecutor exec,
405       final Map<String, SnapshotRegionManifest> regionManifests,
406       final List<HRegionInfo> regions) throws IOException {
407     if (regions == null || regions.size() == 0) return;
408     ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
409       @Override
410       public void editRegion(final HRegionInfo hri) throws IOException {
411         restoreRegion(hri, regionManifests.get(hri.getEncodedName()));
412       }
413     });
414   }
415 
416   /**
417    * Restore specified mob regions by restoring content to the snapshot state.
418    */
419   private void restoreHdfsMobRegions(final ThreadPoolExecutor exec,
420       final Map<String, SnapshotRegionManifest> regionManifests,
421       final List<HRegionInfo> regions) throws IOException {
422     if (regions == null || regions.size() == 0) return;
423     ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
424       @Override
425       public void editRegion(final HRegionInfo hri) throws IOException {
426         restoreMobRegion(hri, regionManifests.get(hri.getEncodedName()));
427       }
428     });
429   }
430 
431   private Map<String, List<SnapshotRegionManifest.StoreFile>> getRegionHFileReferences(
432       final SnapshotRegionManifest manifest) {
433     Map<String, List<SnapshotRegionManifest.StoreFile>> familyMap =
434       new HashMap<String, List<SnapshotRegionManifest.StoreFile>>(manifest.getFamilyFilesCount());
435     for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
436       familyMap.put(familyFiles.getFamilyName().toStringUtf8(),
437         new ArrayList<SnapshotRegionManifest.StoreFile>(familyFiles.getStoreFilesList()));
438     }
439     return familyMap;
440   }
441 
442   /**
443    * Restore region by removing files not in the snapshot
444    * and adding the missing ones from the snapshot.
445    */
446   private void restoreRegion(final HRegionInfo regionInfo,
447       final SnapshotRegionManifest regionManifest) throws IOException {
448     restoreRegion(regionInfo, regionManifest, new Path(tableDir, regionInfo.getEncodedName()));
449   }
450 
451   /**
452    * Restore mob region by removing files not in the snapshot
453    * and adding the missing ones from the snapshot.
454    */
455   private void restoreMobRegion(final HRegionInfo regionInfo,
456       final SnapshotRegionManifest regionManifest) throws IOException {
457     if (regionManifest == null) {
458       return;
459     }
460     restoreRegion(regionInfo, regionManifest,
461       MobUtils.getMobRegionPath(conf, tableDesc.getTableName()));
462   }
463 
464   /**
465    * Restore region by removing files not in the snapshot
466    * and adding the missing ones from the snapshot.
467    */
468   private void restoreRegion(final HRegionInfo regionInfo,
469       final SnapshotRegionManifest regionManifest, Path regionDir) throws IOException {
470     Map<String, List<SnapshotRegionManifest.StoreFile>> snapshotFiles =
471                 getRegionHFileReferences(regionManifest);
472 
473     String tableName = tableDesc.getTableName().getNameAsString();
474 
475     // Restore families present in the table
476     for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
477       byte[] family = Bytes.toBytes(familyDir.getName());
478       Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
479       List<SnapshotRegionManifest.StoreFile> snapshotFamilyFiles =
480           snapshotFiles.remove(familyDir.getName());
481       if (snapshotFamilyFiles != null) {
482         List<SnapshotRegionManifest.StoreFile> hfilesToAdd =
483             new ArrayList<SnapshotRegionManifest.StoreFile>();
484         for (SnapshotRegionManifest.StoreFile storeFile: snapshotFamilyFiles) {
485           if (familyFiles.contains(storeFile.getName())) {
486             // HFile already present
487             familyFiles.remove(storeFile.getName());
488           } else {
489             // HFile missing
490             hfilesToAdd.add(storeFile);
491           }
492         }
493 
494         // Remove hfiles not present in the snapshot
495         for (String hfileName: familyFiles) {
496           Path hfile = new Path(familyDir, hfileName);
497           LOG.trace("Removing hfile=" + hfileName +
498             " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
499           HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
500         }
501 
502         // Restore Missing files
503         for (SnapshotRegionManifest.StoreFile storeFile: hfilesToAdd) {
504           LOG.debug("Adding HFileLink " + storeFile.getName() +
505             " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
506           restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
507         }
508       } else {
509         // Family doesn't exists in the snapshot
510         LOG.trace("Removing family=" + Bytes.toString(family) +
511           " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
512         HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo, familyDir, family);
513         fs.delete(familyDir, true);
514       }
515     }
516 
517     // Add families not present in the table
518     for (Map.Entry<String, List<SnapshotRegionManifest.StoreFile>> familyEntry:
519                                                                       snapshotFiles.entrySet()) {
520       Path familyDir = new Path(regionDir, familyEntry.getKey());
521       if (!fs.mkdirs(familyDir)) {
522         throw new IOException("Unable to create familyDir=" + familyDir);
523       }
524 
525       for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue()) {
526         LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
527         restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
528       }
529     }
530   }
531 
532   /**
533    * @return The set of files in the specified family directory.
534    */
535   private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws IOException {
536     Set<String> familyFiles = new HashSet<String>();
537 
538     FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir);
539     if (hfiles == null) return familyFiles;
540 
541     for (FileStatus hfileRef: hfiles) {
542       String hfileName = hfileRef.getPath().getName();
543       familyFiles.add(hfileName);
544     }
545 
546     return familyFiles;
547   }
548 
549   /**
550    * Clone specified regions. For each region create a new region
551    * and create a HFileLink for each hfile.
552    */
553   private HRegionInfo[] cloneHdfsRegions(final ThreadPoolExecutor exec,
554       final Map<String, SnapshotRegionManifest> regionManifests,
555       final List<HRegionInfo> regions) throws IOException {
556     if (regions == null || regions.size() == 0) return null;
557 
558     final Map<String, HRegionInfo> snapshotRegions =
559       new HashMap<String, HRegionInfo>(regions.size());
560 
561     // clone region info (change embedded tableName with the new one)
562     HRegionInfo[] clonedRegionsInfo = new HRegionInfo[regions.size()];
563     for (int i = 0; i < clonedRegionsInfo.length; ++i) {
564       // clone the region info from the snapshot region info
565       HRegionInfo snapshotRegionInfo = regions.get(i);
566       clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo);
567 
568       // add the region name mapping between snapshot and cloned
569       String snapshotRegionName = snapshotRegionInfo.getEncodedName();
570       String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
571       regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName));
572       LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName);
573 
574       // Add mapping between cloned region name and snapshot region info
575       snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
576     }
577 
578     // create the regions on disk
579     ModifyRegionUtils.createRegions(exec, conf, rootDir, tableDir,
580       tableDesc, clonedRegionsInfo, new ModifyRegionUtils.RegionFillTask() {
581         @Override
582         public void fillRegion(final HRegion region) throws IOException {
583           HRegionInfo snapshotHri = snapshotRegions.get(region.getRegionInfo().getEncodedName());
584           cloneRegion(region, snapshotHri, regionManifests.get(snapshotHri.getEncodedName()));
585         }
586       });
587 
588     return clonedRegionsInfo;
589   }
590 
591   /**
592    * Clone the mob region. For the region create a new region
593    * and create a HFileLink for each hfile.
594    */
595   private void cloneHdfsMobRegion(final Map<String, SnapshotRegionManifest> regionManifests,
596       final HRegionInfo region) throws IOException {
597     // clone region info (change embedded tableName with the new one)
598     Path clonedRegionPath = MobUtils.getMobRegionPath(conf, tableDesc.getTableName());
599     cloneRegion(clonedRegionPath, region, regionManifests.get(region.getEncodedName()));
600   }
601 
602   /**
603    * Clone region directory content from the snapshot info.
604    *
605    * Each region is encoded with the table name, so the cloned region will have
606    * a different region name.
607    *
608    * Instead of copying the hfiles a HFileLink is created.
609    *
610    * @param regionDir {@link Path} cloned dir
611    * @param snapshotRegionInfo
612    */
613   private void cloneRegion(final Path regionDir, final HRegionInfo snapshotRegionInfo,
614       final SnapshotRegionManifest manifest) throws IOException {
615     final String tableName = tableDesc.getTableName().getNameAsString();
616     for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
617       Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
618       for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
619         LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
620         restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
621       }
622     }
623   }
624 
625   /**
626    * Clone region directory content from the snapshot info.
627    *
628    * Each region is encoded with the table name, so the cloned region will have
629    * a different region name.
630    *
631    * Instead of copying the hfiles a HFileLink is created.
632    *
633    * @param region {@link HRegion} cloned
634    * @param snapshotRegionInfo
635    */
636   private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo,
637       final SnapshotRegionManifest manifest) throws IOException {
638     cloneRegion(new Path(tableDir, region.getRegionInfo().getEncodedName()), snapshotRegionInfo,
639       manifest);
640   }
641 
642   /**
643    * Create a new {@link HFileLink} to reference the store file.
644    * <p>The store file in the snapshot can be a simple hfile, an HFileLink or a reference.
645    * <ul>
646    *   <li>hfile: abc -> table=region-abc
647    *   <li>reference: abc.1234 -> table=region-abc.1234
648    *   <li>hfilelink: table=region-hfile -> table=region-hfile
649    * </ul>
650    * @param familyDir destination directory for the store file
651    * @param regionInfo destination region info for the table
652    * @param storeFile store file name (can be a Reference, HFileLink or simple HFile)
653    * @param createBackRef - Whether back reference should be created. Defaults to true.
654    */
655   private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
656       final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef)
657           throws IOException {
658     String hfileName = storeFile.getName();
659     if (HFileLink.isHFileLink(hfileName)) {
660       HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef);
661     } else if (StoreFileInfo.isReference(hfileName)) {
662       restoreReferenceFile(familyDir, regionInfo, storeFile);
663     } else {
664       HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef);
665     }
666   }
667 
668   /**
669    * Create a new {@link Reference} as copy of the source one.
670    * <p><blockquote><pre>
671    * The source table looks like:
672    *    1234/abc      (original file)
673    *    5678/abc.1234 (reference file)
674    *
675    * After the clone operation looks like:
676    *   wxyz/table=1234-abc
677    *   stuv/table=1234-abc.wxyz
678    *
679    * NOTE that the region name in the clone changes (md5 of regioninfo)
680    * and the reference should reflect that change.
681    * </pre></blockquote>
682    * @param familyDir destination directory for the store file
683    * @param regionInfo destination region info for the table
684    * @param storeFile reference file name
685    */
686   private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo,
687       final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
688     String hfileName = storeFile.getName();
689 
690     // Extract the referred information (hfile name and parent region)
691     Path refPath =
692         StoreFileInfo.getReferredToFile(new Path(new Path(new Path(new Path(snapshotTable
693             .getNamespaceAsString(), snapshotTable.getQualifierAsString()), regionInfo
694             .getEncodedName()), familyDir.getName()), hfileName)); 
695     String snapshotRegionName = refPath.getParent().getParent().getName();
696     String fileName = refPath.getName();
697 
698     // The new reference should have the cloned region name as parent, if it is a clone.
699     String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
700     if (clonedRegionName == null) clonedRegionName = snapshotRegionName;
701 
702     // The output file should be a reference link table=snapshotRegion-fileName.clonedRegionName
703     Path linkPath = null;
704     String refLink = fileName;
705     if (!HFileLink.isHFileLink(fileName)) {
706       refLink = HFileLink.createHFileLinkName(snapshotTable, snapshotRegionName, fileName);
707       linkPath = new Path(familyDir,
708         HFileLink.createHFileLinkName(snapshotTable, regionInfo.getEncodedName(), hfileName));
709     }
710 
711     Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
712 
713     // Create the new reference
714     if (storeFile.hasReference()) {
715       Reference reference = Reference.convert(storeFile.getReference());
716       reference.write(fs, outPath);
717     } else {
718       InputStream in;
719       if (linkPath != null) {
720         in = HFileLink.buildFromHFileLinkPattern(conf, linkPath).open(fs);
721       } else {
722         linkPath = new Path(new Path(HRegion.getRegionDir(snapshotManifest.getSnapshotDir(),
723                         regionInfo.getEncodedName()), familyDir.getName()), hfileName);
724         in = fs.open(linkPath);
725       }
726       OutputStream out = fs.create(outPath);
727       IOUtils.copyBytes(in, out, conf);
728     }
729 
730     // Add the daughter region to the map
731     String regionName = Bytes.toString(regionsMap.get(regionInfo.getEncodedNameAsBytes()));
732     LOG.debug("Restore reference " + regionName + " to " + clonedRegionName);
733     synchronized (parentsMap) {
734       Pair<String, String> daughters = parentsMap.get(clonedRegionName);
735       if (daughters == null) {
736         daughters = new Pair<String, String>(regionName, null);
737         parentsMap.put(clonedRegionName, daughters);
738       } else if (!regionName.equals(daughters.getFirst())) {
739         daughters.setSecond(regionName);
740       }
741     }
742   }
743 
744   /**
745    * Create a new {@link HRegionInfo} from the snapshot region info.
746    * Keep the same startKey, endKey, regionId and split information but change
747    * the table name.
748    *
749    * @param snapshotRegionInfo Info for region to clone.
750    * @return the new HRegion instance
751    */
752   public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) {
753     HRegionInfo regionInfo = new HRegionInfo(tableDesc.getTableName(),
754                       snapshotRegionInfo.getStartKey(), snapshotRegionInfo.getEndKey(),
755                       snapshotRegionInfo.isSplit(), snapshotRegionInfo.getRegionId());
756     regionInfo.setOffline(snapshotRegionInfo.isOffline());
757     return regionInfo;
758   }
759 
760   /**
761    * @return the set of the regions contained in the table
762    */
763   private List<HRegionInfo> getTableRegions() throws IOException {
764     LOG.debug("get table regions: " + tableDir);
765     FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
766     if (regionDirs == null) return null;
767 
768     List<HRegionInfo> regions = new LinkedList<HRegionInfo>();
769     for (FileStatus regionDir: regionDirs) {
770       HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir.getPath());
771       regions.add(hri);
772     }
773     LOG.debug("found " + regions.size() + " regions for table=" +
774         tableDesc.getTableName().getNameAsString());
775     return regions;
776   }
777 
778   /**
779    * Create a new table descriptor cloning the snapshot table schema.
780    *
781    * @param snapshotTableDescriptor
782    * @param tableName
783    * @return cloned table descriptor
784    * @throws IOException
785    */
786   public static HTableDescriptor cloneTableSchema(final HTableDescriptor snapshotTableDescriptor,
787       final TableName tableName) throws IOException {
788     HTableDescriptor htd = new HTableDescriptor(tableName);
789     for (HColumnDescriptor hcd: snapshotTableDescriptor.getColumnFamilies()) {
790       htd.addFamily(hcd);
791     }
792     for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e:
793         snapshotTableDescriptor.getValues().entrySet()) {
794       htd.setValue(e.getKey(), e.getValue());
795     }
796     for (Map.Entry<String, String> e: snapshotTableDescriptor.getConfiguration().entrySet()) {
797       htd.setConfiguration(e.getKey(), e.getValue());
798     }
799     return htd;
800   }
801 
802   /**
803    * Copy the snapshot files for a snapshot scanner, discards meta changes.
804    * @param conf
805    * @param fs
806    * @param rootDir
807    * @param restoreDir
808    * @param snapshotName
809    * @throws IOException
810    */
811   public static void copySnapshotForScanner(Configuration conf, FileSystem fs, Path rootDir,
812       Path restoreDir, String snapshotName) throws IOException {
813     // ensure that restore dir is not under root dir
814     if (!restoreDir.getFileSystem(conf).getUri().equals(rootDir.getFileSystem(conf).getUri())) {
815       throw new IllegalArgumentException("Filesystems for restore directory (" +
816           restoreDir.getFileSystem(conf).getUri() + ") and HBase root directory (" +
817           rootDir.getFileSystem(conf).getUri() + ") should be the same");
818     }
819     if (restoreDir.toUri().getPath().startsWith(rootDir.toUri().getPath())) {
820       throw new IllegalArgumentException("Restore directory cannot be a sub directory of HBase " +
821           "root directory. RootDir: " + rootDir + ", restoreDir: " + restoreDir);
822     }
823 
824     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
825     SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
826     SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
827 
828     MonitoredTask status = TaskMonitor.get().createStatus(
829         "Restoring  snapshot '" + snapshotName + "' to directory " + restoreDir);
830     ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
831 
832     // we send createBackRefs=false so that restored hfiles do not create back reference links
833     // in the base hbase root dir.
834     RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs,
835       manifest, manifest.getTableDescriptor(), restoreDir, monitor, status, false);
836     helper.restoreHdfsRegions(); // TODO: parallelize.
837 
838     if (LOG.isDebugEnabled()) {
839       LOG.debug("Restored table dir:" + restoreDir);
840       FSUtils.logFileSystemState(fs, restoreDir, LOG);
841     }
842   }
843 
844   public static void restoreSnapshotACL(SnapshotDescription snapshot, TableName newTableName,
845       Configuration conf) throws IOException {
846     if (snapshot.hasUsersAndPermissions() && snapshot.getUsersAndPermissions() != null) {
847       LOG.info("Restore snapshot acl to table. snapshot: " + snapshot + ", table: " + newTableName);
848       ListMultimap<String, TablePermission> perms =
849           ProtobufUtil.toUserTablePermissions(snapshot.getUsersAndPermissions());
850       try {
851         for (Entry<String, TablePermission> e : perms.entries()) {
852           String user = e.getKey();
853           TablePermission perm = e.getValue();
854           perm.setTableName(newTableName);
855           AccessControlClient.grant(conf, perm.getTableName(), user, perm.getFamily(),
856             perm.getQualifier(), perm.getActions());
857         }
858       } catch (Throwable e) {
859         throw new IOException("Grant acl into newly creatd table failed. snapshot: " + snapshot
860             + ", table: " + newTableName, e);
861       }
862     }
863   }
864 }