View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.snapshot;
19  
20  import java.io.IOException;
21  import java.util.List;
22  import java.util.Map;
23  import java.util.Set;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.classification.InterfaceAudience;
28  import org.apache.hadoop.classification.InterfaceStability;
29  import org.apache.hadoop.fs.FileSystem;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.TableName;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.HTableDescriptor;
34  import org.apache.hadoop.hbase.ServerName;
35  import org.apache.hadoop.hbase.catalog.MetaReader;
36  import org.apache.hadoop.hbase.master.MasterServices;
37  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
38  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
39  import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
40  import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
41  import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
42  import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
43  import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
44  
45  /**
46   * General snapshot verification on the master.
47   * <p>
48   * This is a light-weight verification mechanism for all the files in a snapshot. It doesn't
49   * attempt to verify that the files are exact copies (that would be paramount to taking the
50   * snapshot again!), but instead just attempts to ensure that the files match the expected
51   * files and are the same length.
52   * <p>
53   * Taking an online snapshots can race against other operations and this is an last line of
54   * defense.  For example, if meta changes between when snapshots are taken not all regions of a
55   * table may be present.  This can be caused by a region split (daughters present on this scan,
56   * but snapshot took parent), or move (snapshots only checks lists of region servers, a move could
57   * have caused a region to be skipped or done twice).
58   * <p>
59   * Current snapshot files checked:
60   * <ol>
61   * <li>SnapshotDescription is readable</li>
62   * <li>Table info is readable</li>
63   * <li>Regions</li>
64   * <ul>
65   * <li>Matching regions in the snapshot as currently in the table</li>
66   * <li>{@link HRegionInfo} matches the current and stored regions</li>
67   * <li>All referenced hfiles have valid names</li>
68   * <li>All the hfiles are present (either in .archive directory in the region)</li>
69   * <li>All recovered.edits files are present (by name) and have the correct file size</li>
70   * </ul>
71   * </ol>
72   */
73  @InterfaceAudience.Private
74  @InterfaceStability.Unstable
75  public final class MasterSnapshotVerifier {
76    private static final Log LOG = LogFactory.getLog(MasterSnapshotVerifier.class);
77  
78    private SnapshotDescription snapshot;
79    private FileSystem fs;
80    private Path rootDir;
81    private TableName tableName;
82    private MasterServices services;
83  
84    /**
85     * @param services services for the master
86     * @param snapshot snapshot to check
87     * @param rootDir root directory of the hbase installation.
88     */
89    public MasterSnapshotVerifier(MasterServices services, SnapshotDescription snapshot, Path rootDir) {
90      this.fs = services.getMasterFileSystem().getFileSystem();
91      this.services = services;
92      this.snapshot = snapshot;
93      this.rootDir = rootDir;
94      this.tableName = TableName.valueOf(snapshot.getTable());
95    }
96  
97    /**
98     * Verify that the snapshot in the directory is a valid snapshot
99     * @param snapshotDir snapshot directory to check
100    * @param snapshotServers {@link ServerName} of the servers that are involved in the snapshot
101    * @throws CorruptedSnapshotException if the snapshot is invalid
102    * @throws IOException if there is an unexpected connection issue to the filesystem
103    */
104   public void verifySnapshot(Path snapshotDir, Set<String> snapshotServers)
105       throws CorruptedSnapshotException, IOException {
106     SnapshotManifest manifest = SnapshotManifest.open(services.getConfiguration(), fs,
107                                                       snapshotDir, snapshot);
108     // verify snapshot info matches
109     verifySnapshotDescription(snapshotDir);
110 
111     // check that tableinfo is a valid table description
112     verifyTableInfo(manifest);
113 
114     // check that each region is valid
115     verifyRegions(manifest);
116   }
117 
118   /**
119    * Check that the snapshot description written in the filesystem matches the current snapshot
120    * @param snapshotDir snapshot directory to check
121    */
122   private void verifySnapshotDescription(Path snapshotDir) throws CorruptedSnapshotException {
123     SnapshotDescription found = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
124     if (!this.snapshot.equals(found)) {
125       throw new CorruptedSnapshotException("Snapshot read (" + found
126           + ") doesn't equal snapshot we ran (" + snapshot + ").", snapshot);
127     }
128   }
129 
130   /**
131    * Check that the table descriptor for the snapshot is a valid table descriptor
132    * @param manifest snapshot manifest to inspect
133    */
134   private void verifyTableInfo(final SnapshotManifest manifest) throws IOException {
135     HTableDescriptor htd = manifest.getTableDescriptor();
136     if (htd == null) {
137       throw new CorruptedSnapshotException("Missing Table Descriptor", snapshot);
138     }
139 
140     if (!htd.getNameAsString().equals(snapshot.getTable())) {
141       throw new CorruptedSnapshotException("Invalid Table Descriptor. Expected "
142         + snapshot.getTable() + " name, got " + htd.getNameAsString(), snapshot);
143     }
144   }
145 
146   /**
147    * Check that all the regions in the snapshot are valid, and accounted for.
148    * @param manifest snapshot manifest to inspect
149    * @throws IOException if we can't reach hbase:meta or read the files from the FS
150    */
151   private void verifyRegions(final SnapshotManifest manifest) throws IOException {
152     List<HRegionInfo> regions = MetaReader.getTableRegions(this.services.getCatalogTracker(),
153         tableName);
154 
155     Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
156     if (regionManifests == null) {
157       String msg = "Snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " looks empty";
158       LOG.error(msg);
159       throw new CorruptedSnapshotException(msg);
160     }
161 
162     String errorMsg = "";
163     if (regionManifests.size() != regions.size()) {
164       errorMsg = "Regions moved during the snapshot '" +
165                    ClientSnapshotDescriptionUtils.toString(snapshot) + "'. expected=" +
166                    regions.size() + " snapshotted=" + regionManifests.size() + ".";
167       LOG.error(errorMsg);
168     }
169 
170     // Verify HRegionInfo
171     for (HRegionInfo region : regions) {
172       SnapshotRegionManifest regionManifest = regionManifests.get(region.getEncodedName());
173       if (regionManifest == null) {
174         // could happen due to a move or split race.
175         String mesg = " No snapshot region directory found for region:" + region;
176         if (errorMsg.isEmpty()) errorMsg = mesg;
177         LOG.error(mesg);
178         continue;
179       }
180 
181       verifyRegionInfo(region, regionManifest);
182     }
183 
184     if (!errorMsg.isEmpty()) {
185       throw new CorruptedSnapshotException(errorMsg);
186     }
187 
188     // Verify Snapshot HFiles
189     SnapshotReferenceUtil.verifySnapshot(services.getConfiguration(), fs, manifest);
190   }
191 
192   /**
193    * Verify that the regionInfo is valid
194    * @param region the region to check
195    * @param manifest snapshot manifest to inspect
196    */
197   private void verifyRegionInfo(final HRegionInfo region,
198       final SnapshotRegionManifest manifest) throws IOException {
199     HRegionInfo manifestRegionInfo = HRegionInfo.convert(manifest.getRegionInfo());
200     if (!region.equals(manifestRegionInfo)) {
201       String msg = "Manifest region info " + manifestRegionInfo +
202                    "doesn't match expected region:" + region;
203       throw new CorruptedSnapshotException(msg, snapshot);
204     }
205   }
206 }