001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.junit.Assert.fail;
021
022import java.io.IOException;
023import org.apache.hadoop.conf.Configuration;
024import org.apache.hadoop.fs.FSDataOutputStream;
025import org.apache.hadoop.fs.FileSystem;
026import org.apache.hadoop.fs.Path;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseTestingUtility;
029import org.apache.hadoop.hbase.HRegionInfo;
030import org.apache.hadoop.hbase.TableName;
031import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
032import org.apache.hadoop.hbase.testclassification.MasterTests;
033import org.apache.hadoop.hbase.testclassification.MediumTests;
034import org.apache.hadoop.hbase.util.Bytes;
035import org.junit.After;
036import org.junit.Before;
037import org.junit.ClassRule;
038import org.junit.Test;
039import org.junit.experimental.categories.Category;
040import org.slf4j.Logger;
041import org.slf4j.LoggerFactory;
042
043import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
044
045import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
046import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDataManifest;
047import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
048import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
049
050@Category({ MasterTests.class, MediumTests.class })
051public class TestSnapshotManifest {
052
053  @ClassRule
054  public static final HBaseClassTestRule CLASS_RULE =
055    HBaseClassTestRule.forClass(TestSnapshotManifest.class);
056
057  private final Logger LOG = LoggerFactory.getLogger(getClass());
058
059  private static final String TABLE_NAME_STR = "testSnapshotManifest";
060  private static final TableName TABLE_NAME = TableName.valueOf(TABLE_NAME_STR);
061  private static final int TEST_NUM_REGIONS = 16000;
062  private static final int TEST_NUM_REGIONFILES = 1000000;
063
064  private static HBaseTestingUtility TEST_UTIL;
065  private Configuration conf;
066  private FileSystem fs;
067  private Path rootDir;
068  private Path snapshotDir;
069  private SnapshotDescription snapshotDesc;
070  private SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder;
071
072  @Before
073  public void setup() throws Exception {
074    TEST_UTIL = HBaseTestingUtility.createLocalHTU();
075
076    rootDir = TEST_UTIL.getDataTestDir(TABLE_NAME_STR);
077    fs = TEST_UTIL.getTestFileSystem();
078    conf = TEST_UTIL.getConfiguration();
079
080    SnapshotTestingUtils.SnapshotMock snapshotMock =
081      new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir);
082    builder = snapshotMock.createSnapshotV2("snapshot", TABLE_NAME_STR, 0);
083    snapshotDir = builder.commit();
084    snapshotDesc = builder.getSnapshotDescription();
085  }
086
087  @After
088  public void tearDown() throws Exception {
089    fs.delete(rootDir, true);
090  }
091
092  @Test
093  public void testReadSnapshotManifest() throws IOException {
094
095    Path p = createDataManifest();
096    try {
097      SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
098      fail("fail to test snapshot manifest because message size is too small.");
099    } catch (CorruptedSnapshotException cse) {
100      try {
101        conf.setInt(SnapshotManifest.SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 128 * 1024 * 1024);
102        SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
103        LOG.info("open snapshot manifest succeed.");
104      } catch (CorruptedSnapshotException cse2) {
105        fail("fail to take snapshot because Manifest proto-message too large.");
106      }
107    } finally {
108      fs.delete(p, false);
109    }
110  }
111
112  @Test
113  public void testReadSnapshotRegionManifest() throws IOException {
114
115    // remove datamanifest file
116    fs.delete(new Path(snapshotDir, SnapshotManifest.DATA_MANIFEST_NAME), true);
117    Path regionPath = createRegionManifest();
118
119    try {
120      conf.setInt(SnapshotManifest.SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 128 * 1024 * 1024);
121      SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
122    } catch (CorruptedSnapshotException e) {
123      fail("fail to test snapshot manifest because region message size is too small.");
124    } finally {
125      fs.delete(regionPath, false);
126    }
127  }
128
129  private Path createDataManifest() throws IOException {
130    SnapshotDataManifest.Builder dataManifestBuilder = SnapshotDataManifest.newBuilder();
131    byte[] startKey = null;
132    byte[] stopKey = null;
133    for (int i = 1; i <= TEST_NUM_REGIONS; i++) {
134      stopKey = Bytes.toBytes(String.format("%016d", i));
135      HRegionInfo regionInfo = new HRegionInfo(TABLE_NAME, startKey, stopKey, false);
136      SnapshotRegionManifest.Builder dataRegionManifestBuilder =
137        SnapshotRegionManifest.newBuilder();
138
139      for (ColumnFamilyDescriptor hcd : builder.getTableDescriptor().getColumnFamilies()) {
140        SnapshotRegionManifest.FamilyFiles.Builder family =
141          SnapshotRegionManifest.FamilyFiles.newBuilder();
142        family.setFamilyName(UnsafeByteOperations.unsafeWrap(hcd.getName()));
143        for (int j = 0; j < 100; ++j) {
144          SnapshotRegionManifest.StoreFile.Builder sfManifest =
145            SnapshotRegionManifest.StoreFile.newBuilder();
146          sfManifest.setName(String.format("%032d", i));
147          sfManifest.setFileSize((1 + i) * (1 + i) * 1024);
148          family.addStoreFiles(sfManifest.build());
149        }
150        dataRegionManifestBuilder.addFamilyFiles(family.build());
151      }
152
153      dataRegionManifestBuilder.setRegionInfo(HRegionInfo.convert(regionInfo));
154      dataManifestBuilder.addRegionManifests(dataRegionManifestBuilder.build());
155
156      startKey = stopKey;
157    }
158
159    dataManifestBuilder.setTableSchema(ProtobufUtil.toTableSchema(builder.getTableDescriptor()));
160
161    SnapshotDataManifest dataManifest = dataManifestBuilder.build();
162    return writeDataManifest(dataManifest);
163  }
164
165  private Path createRegionManifest() throws IOException {
166    byte[] startKey = Bytes.toBytes("AAAAAA");
167    byte[] stopKey = Bytes.toBytes("BBBBBB");
168    HRegionInfo regionInfo = new HRegionInfo(TABLE_NAME, startKey, stopKey, false);
169    SnapshotRegionManifest.Builder dataRegionManifestBuilder = SnapshotRegionManifest.newBuilder();
170    dataRegionManifestBuilder.setRegionInfo(HRegionInfo.convert(regionInfo));
171
172    for (ColumnFamilyDescriptor hcd : builder.getTableDescriptor().getColumnFamilies()) {
173      SnapshotRegionManifest.FamilyFiles.Builder family =
174        SnapshotRegionManifest.FamilyFiles.newBuilder();
175      family.setFamilyName(UnsafeByteOperations.unsafeWrap(hcd.getName()));
176      for (int j = 0; j < TEST_NUM_REGIONFILES; ++j) {
177        SnapshotRegionManifest.StoreFile.Builder sfManifest =
178          SnapshotRegionManifest.StoreFile.newBuilder();
179        sfManifest.setName(String.format("%064d", j));
180        sfManifest.setFileSize(j * 1024);
181        family.addStoreFiles(sfManifest.build());
182      }
183      dataRegionManifestBuilder.addFamilyFiles(family.build());
184    }
185
186    SnapshotRegionManifest manifest = dataRegionManifestBuilder.build();
187    Path regionPath = new Path(snapshotDir,
188      SnapshotManifestV2.SNAPSHOT_MANIFEST_PREFIX + regionInfo.getEncodedName());
189
190    FSDataOutputStream stream = fs.create(regionPath);
191    try {
192      manifest.writeTo(stream);
193    } finally {
194      stream.close();
195    }
196
197    return regionPath;
198  }
199
200  private Path writeDataManifest(final SnapshotDataManifest manifest) throws IOException {
201    Path dataRegionPath = new Path(snapshotDir, SnapshotManifest.DATA_MANIFEST_NAME);
202    FSDataOutputStream stream = fs.create(dataRegionPath);
203    try {
204      manifest.writeTo(stream);
205    } finally {
206      stream.close();
207    }
208
209    return dataRegionPath;
210  }
211}