View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.io.InterruptedIOException;
23  import java.util.ArrayList;
24  import java.util.List;
25  import java.util.concurrent.Callable;
26  import java.util.concurrent.Executor;
27  import java.util.concurrent.ExecutionException;
28  import java.util.concurrent.ExecutorCompletionService;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.hbase.classification.InterfaceAudience;
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.fs.FSDataInputStream;
35  import org.apache.hadoop.fs.FSDataOutputStream;
36  import org.apache.hadoop.fs.FileStatus;
37  import org.apache.hadoop.fs.FileSystem;
38  import org.apache.hadoop.fs.Path;
39  import org.apache.hadoop.fs.PathFilter;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
42  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
43  import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
44  import org.apache.hadoop.hbase.util.FSUtils;
45  
46  import com.google.protobuf.HBaseZeroCopyByteString;
47  
48  /**
49   * DO NOT USE DIRECTLY. USE {@link SnapshotManifest}.
50   *
51   * Snapshot v2 layout format
52   *  - Single Manifest file containing all the information of regions
53   *  - In the online-snapshot case each region will write a "region manifest"
54   *      /snapshotName/manifest.regionName
55   */
56  @InterfaceAudience.Private
57  public class SnapshotManifestV2 {
58    private static final Log LOG = LogFactory.getLog(SnapshotManifestV2.class);
59  
60    public static final int DESCRIPTOR_VERSION = 2;
61  
62    private static final String SNAPSHOT_MANIFEST_PREFIX = "region-manifest.";
63  
64    static class ManifestBuilder implements SnapshotManifest.RegionVisitor<
65                      SnapshotRegionManifest.Builder, SnapshotRegionManifest.FamilyFiles.Builder> {
66      private final Configuration conf;
67      private final Path snapshotDir;
68      private final FileSystem fs;
69  
70      public ManifestBuilder(final Configuration conf, final FileSystem fs, final Path snapshotDir) {
71        this.snapshotDir = snapshotDir;
72        this.conf = conf;
73        this.fs = fs;
74      }
75  
76      public SnapshotRegionManifest.Builder regionOpen(final HRegionInfo regionInfo) {
77        SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
78        manifest.setRegionInfo(HRegionInfo.convert(regionInfo));
79        return manifest;
80      }
81  
82      public void regionClose(final SnapshotRegionManifest.Builder region) throws IOException {
83        SnapshotRegionManifest manifest = region.build();
84        FSDataOutputStream stream = fs.create(getRegionManifestPath(snapshotDir, manifest));
85        try {
86          manifest.writeTo(stream);
87        } finally {
88          stream.close();
89        }
90      }
91  
92      public SnapshotRegionManifest.FamilyFiles.Builder familyOpen(
93          final SnapshotRegionManifest.Builder region, final byte[] familyName) {
94        SnapshotRegionManifest.FamilyFiles.Builder family =
95            SnapshotRegionManifest.FamilyFiles.newBuilder();
96        family.setFamilyName(HBaseZeroCopyByteString.wrap(familyName));
97        return family;
98      }
99  
100     public void familyClose(final SnapshotRegionManifest.Builder region,
101         final SnapshotRegionManifest.FamilyFiles.Builder family) {
102       region.addFamilyFiles(family.build());
103     }
104 
105     public void storeFile(final SnapshotRegionManifest.Builder region,
106         final SnapshotRegionManifest.FamilyFiles.Builder family, final StoreFileInfo storeFile)
107         throws IOException {
108       SnapshotRegionManifest.StoreFile.Builder sfManifest =
109             SnapshotRegionManifest.StoreFile.newBuilder();
110       sfManifest.setName(storeFile.getPath().getName());
111       if (storeFile.isReference()) {
112         sfManifest.setReference(storeFile.getReference().convert());
113       }
114       sfManifest.setFileSize(storeFile.getReferencedFileStatus(fs).getLen());
115       family.addStoreFiles(sfManifest.build());
116     }
117   }
118 
119   static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf,
120       final Executor executor,final FileSystem fs, final Path snapshotDir,
121       final SnapshotDescription desc) throws IOException {
122     FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() {
123       @Override
124       public boolean accept(Path path) {
125         return path.getName().startsWith(SNAPSHOT_MANIFEST_PREFIX);
126       }
127     });
128 
129     if (manifestFiles == null || manifestFiles.length == 0) return null;
130 
131     final ExecutorCompletionService<SnapshotRegionManifest> completionService =
132       new ExecutorCompletionService<SnapshotRegionManifest>(executor);
133     for (final FileStatus st: manifestFiles) {
134       completionService.submit(new Callable<SnapshotRegionManifest>() {
135         @Override
136         public SnapshotRegionManifest call() throws IOException {
137           FSDataInputStream stream = fs.open(st.getPath());
138           try {
139             return SnapshotRegionManifest.parseFrom(stream);
140           } finally {
141             stream.close();
142           }
143         }
144       });
145     }
146 
147     ArrayList<SnapshotRegionManifest> regionsManifest =
148         new ArrayList<SnapshotRegionManifest>(manifestFiles.length);
149     try {
150       for (int i = 0; i < manifestFiles.length; ++i) {
151         regionsManifest.add(completionService.take().get());
152       }
153     } catch (InterruptedException e) {
154       throw new InterruptedIOException(e.getMessage());
155     } catch (ExecutionException e) {
156       IOException ex = new IOException();
157       ex.initCause(e.getCause());
158       throw ex;
159     }
160     return regionsManifest;
161   }
162 
163   static void deleteRegionManifest(final FileSystem fs, final Path snapshotDir,
164       final SnapshotRegionManifest manifest) throws IOException {
165     fs.delete(getRegionManifestPath(snapshotDir, manifest), true);
166   }
167 
168   private static Path getRegionManifestPath(final Path snapshotDir,
169       final SnapshotRegionManifest manifest) {
170     String regionName = SnapshotManifest.getRegionNameFromManifest(manifest);
171     return new Path(snapshotDir, SNAPSHOT_MANIFEST_PREFIX + regionName);
172   }
173 }