001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.junit.Assert.fail;
021
022import java.io.IOException;
023import org.apache.hadoop.conf.Configuration;
024import org.apache.hadoop.fs.FSDataOutputStream;
025import org.apache.hadoop.fs.FileSystem;
026import org.apache.hadoop.fs.Path;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseTestingUtil;
029import org.apache.hadoop.hbase.TableName;
030import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
031import org.apache.hadoop.hbase.client.RegionInfo;
032import org.apache.hadoop.hbase.client.RegionInfoBuilder;
033import org.apache.hadoop.hbase.testclassification.MasterTests;
034import org.apache.hadoop.hbase.testclassification.MediumTests;
035import org.apache.hadoop.hbase.util.Bytes;
036import org.junit.After;
037import org.junit.Before;
038import org.junit.ClassRule;
039import org.junit.Test;
040import org.junit.experimental.categories.Category;
041import org.slf4j.Logger;
042import org.slf4j.LoggerFactory;
043
044import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
045
046import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
047import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDataManifest;
048import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
049import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
050
051@Category({ MasterTests.class, MediumTests.class })
052public class TestSnapshotManifest {
053
054  @ClassRule
055  public static final HBaseClassTestRule CLASS_RULE =
056    HBaseClassTestRule.forClass(TestSnapshotManifest.class);
057
058  private final Logger LOG = LoggerFactory.getLogger(getClass());
059
060  private static final String TABLE_NAME_STR = "testSnapshotManifest";
061  private static final TableName TABLE_NAME = TableName.valueOf(TABLE_NAME_STR);
062  private static final int TEST_NUM_REGIONS = 16000;
063  private static final int TEST_NUM_REGIONFILES = 1000000;
064
065  private static HBaseTestingUtil TEST_UTIL;
066  private Configuration conf;
067  private FileSystem fs;
068  private Path rootDir;
069  private Path snapshotDir;
070  private SnapshotDescription snapshotDesc;
071  private SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder;
072
073  @Before
074  public void setup() throws Exception {
075    TEST_UTIL = new HBaseTestingUtil();
076
077    rootDir = TEST_UTIL.getDataTestDir(TABLE_NAME_STR);
078    fs = TEST_UTIL.getTestFileSystem();
079    conf = TEST_UTIL.getConfiguration();
080
081    SnapshotTestingUtils.SnapshotMock snapshotMock =
082      new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir);
083    builder = snapshotMock.createSnapshotV2("snapshot", TABLE_NAME_STR, 0);
084    snapshotDir = builder.commit();
085    snapshotDesc = builder.getSnapshotDescription();
086  }
087
088  @After
089  public void tearDown() throws Exception {
090    fs.delete(rootDir, true);
091  }
092
093  @Test
094  public void testReadSnapshotManifest() throws IOException {
095
096    Path p = createDataManifest();
097    try {
098      SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
099      fail("fail to test snapshot manifest because message size is too small.");
100    } catch (CorruptedSnapshotException cse) {
101      try {
102        conf.setInt(SnapshotManifest.SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 128 * 1024 * 1024);
103        SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
104        LOG.info("open snapshot manifest succeed.");
105      } catch (CorruptedSnapshotException cse2) {
106        fail("fail to take snapshot because Manifest proto-message too large.");
107      }
108    } finally {
109      fs.delete(p, false);
110    }
111  }
112
113  @Test
114  public void testReadSnapshotRegionManifest() throws IOException {
115
116    // remove datamanifest file
117    fs.delete(new Path(snapshotDir, SnapshotManifest.DATA_MANIFEST_NAME), true);
118    Path regionPath = createRegionManifest();
119
120    try {
121      conf.setInt(SnapshotManifest.SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 128 * 1024 * 1024);
122      SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
123    } catch (CorruptedSnapshotException e) {
124      fail("fail to test snapshot manifest because region message size is too small.");
125    } finally {
126      fs.delete(regionPath, false);
127    }
128  }
129
130  private Path createDataManifest() throws IOException {
131    SnapshotDataManifest.Builder dataManifestBuilder = SnapshotDataManifest.newBuilder();
132    byte[] startKey = null;
133    byte[] stopKey = null;
134    for (int i = 1; i <= TEST_NUM_REGIONS; i++) {
135      stopKey = Bytes.toBytes(String.format("%016d", i));
136      RegionInfo regionInfo = RegionInfoBuilder.newBuilder(TABLE_NAME).build();
137      SnapshotRegionManifest.Builder dataRegionManifestBuilder =
138        SnapshotRegionManifest.newBuilder();
139
140      for (ColumnFamilyDescriptor hcd : builder.getTableDescriptor().getColumnFamilies()) {
141        SnapshotRegionManifest.FamilyFiles.Builder family =
142          SnapshotRegionManifest.FamilyFiles.newBuilder();
143        family.setFamilyName(UnsafeByteOperations.unsafeWrap(hcd.getName()));
144        for (int j = 0; j < 100; ++j) {
145          SnapshotRegionManifest.StoreFile.Builder sfManifest =
146            SnapshotRegionManifest.StoreFile.newBuilder();
147          sfManifest.setName(String.format("%032d", i));
148          sfManifest.setFileSize((1 + i) * (1 + i) * 1024);
149          family.addStoreFiles(sfManifest.build());
150        }
151        dataRegionManifestBuilder.addFamilyFiles(family.build());
152      }
153
154      dataRegionManifestBuilder.setRegionInfo(ProtobufUtil.toRegionInfo(regionInfo));
155      dataManifestBuilder.addRegionManifests(dataRegionManifestBuilder.build());
156
157      startKey = stopKey;
158    }
159
160    dataManifestBuilder.setTableSchema(ProtobufUtil.toTableSchema(builder.getTableDescriptor()));
161
162    SnapshotDataManifest dataManifest = dataManifestBuilder.build();
163    return writeDataManifest(dataManifest);
164  }
165
166  private Path createRegionManifest() throws IOException {
167    byte[] startKey = Bytes.toBytes("AAAAAA");
168    byte[] stopKey = Bytes.toBytes("BBBBBB");
169    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(TABLE_NAME).build();
170    SnapshotRegionManifest.Builder dataRegionManifestBuilder = SnapshotRegionManifest.newBuilder();
171    dataRegionManifestBuilder.setRegionInfo(ProtobufUtil.toRegionInfo(regionInfo));
172
173    for (ColumnFamilyDescriptor hcd : builder.getTableDescriptor().getColumnFamilies()) {
174      SnapshotRegionManifest.FamilyFiles.Builder family =
175        SnapshotRegionManifest.FamilyFiles.newBuilder();
176      family.setFamilyName(UnsafeByteOperations.unsafeWrap(hcd.getName()));
177      for (int j = 0; j < TEST_NUM_REGIONFILES; ++j) {
178        SnapshotRegionManifest.StoreFile.Builder sfManifest =
179          SnapshotRegionManifest.StoreFile.newBuilder();
180        sfManifest.setName(String.format("%064d", j));
181        sfManifest.setFileSize(j * 1024);
182        family.addStoreFiles(sfManifest.build());
183      }
184      dataRegionManifestBuilder.addFamilyFiles(family.build());
185    }
186
187    SnapshotRegionManifest manifest = dataRegionManifestBuilder.build();
188    Path regionPath = new Path(snapshotDir,
189      SnapshotManifestV2.SNAPSHOT_MANIFEST_PREFIX + regionInfo.getEncodedName());
190
191    FSDataOutputStream stream = fs.create(regionPath);
192    try {
193      manifest.writeTo(stream);
194    } finally {
195      stream.close();
196    }
197
198    return regionPath;
199  }
200
201  private Path writeDataManifest(final SnapshotDataManifest manifest) throws IOException {
202    Path dataRegionPath = new Path(snapshotDir, SnapshotManifest.DATA_MANIFEST_NAME);
203    FSDataOutputStream stream = fs.create(dataRegionPath);
204    try {
205      manifest.writeTo(stream);
206    } finally {
207      stream.close();
208    }
209
210    return dataRegionPath;
211  }
212}