001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.junit.Assert.fail;
021
022import java.io.IOException;
023import org.apache.hadoop.conf.Configuration;
024import org.apache.hadoop.fs.FSDataOutputStream;
025import org.apache.hadoop.fs.FileSystem;
026import org.apache.hadoop.fs.Path;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseTestingUtility;
029import org.apache.hadoop.hbase.HRegionInfo;
030import org.apache.hadoop.hbase.TableName;
031import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
032import org.apache.hadoop.hbase.testclassification.MasterTests;
033import org.apache.hadoop.hbase.testclassification.MediumTests;
034import org.apache.hadoop.hbase.util.Bytes;
035import org.junit.After;
036import org.junit.Before;
037import org.junit.ClassRule;
038import org.junit.Test;
039import org.junit.experimental.categories.Category;
040import org.slf4j.Logger;
041import org.slf4j.LoggerFactory;
042
043import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
044
045import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
046import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDataManifest;
047import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
048import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
049
050@Category({MasterTests.class, MediumTests.class})
051public class TestSnapshotManifest {
052
053  @ClassRule
054  public static final HBaseClassTestRule CLASS_RULE =
055      HBaseClassTestRule.forClass(TestSnapshotManifest.class);
056
057  private final Logger LOG = LoggerFactory.getLogger(getClass());
058
059  private static final String TABLE_NAME_STR = "testSnapshotManifest";
060  private static final TableName TABLE_NAME = TableName.valueOf(TABLE_NAME_STR);
061  private static final int TEST_NUM_REGIONS = 16000;
062  private static final int TEST_NUM_REGIONFILES = 1000000;
063
064  private static HBaseTestingUtility TEST_UTIL;
065  private Configuration conf;
066  private FileSystem fs;
067  private Path rootDir;
068  private Path snapshotDir;
069  private SnapshotDescription snapshotDesc;
070  private SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder;
071
072  @Before
073  public void setup() throws Exception {
074    TEST_UTIL = HBaseTestingUtility.createLocalHTU();
075
076    rootDir = TEST_UTIL.getDataTestDir(TABLE_NAME_STR);
077    fs = TEST_UTIL.getTestFileSystem();
078    conf = TEST_UTIL.getConfiguration();
079
080    SnapshotTestingUtils.SnapshotMock snapshotMock =
081      new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir);
082    builder = snapshotMock.createSnapshotV2("snapshot", TABLE_NAME_STR, 0);
083    snapshotDir = builder.commit();
084    snapshotDesc = builder.getSnapshotDescription();
085  }
086
087  @After
088  public void tearDown() throws Exception {
089    fs.delete(rootDir,true);
090  }
091
092  @Test
093  public void testReadSnapshotManifest() throws IOException {
094
095    Path p = createDataManifest();
096    try {
097      SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
098      fail("fail to test snapshot manifest because message size is too small.");
099    } catch (CorruptedSnapshotException cse) {
100      try {
101        conf.setInt(SnapshotManifest.SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 128 * 1024 * 1024);
102        SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
103        LOG.info("open snapshot manifest succeed.");
104      } catch (CorruptedSnapshotException cse2) {
105        fail("fail to take snapshot because Manifest proto-message too large.");
106      }
107    } finally {
108      fs.delete(p, false);
109    }
110  }
111
112  @Test
113  public void testReadSnapshotRegionManifest() throws IOException {
114
115    // remove datamanifest file
116    fs.delete(new Path(snapshotDir, SnapshotManifest.DATA_MANIFEST_NAME), true);
117    Path regionPath = createRegionManifest();
118
119    try {
120      conf.setInt(SnapshotManifest.SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 128 * 1024 * 1024);
121      SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
122    } catch (CorruptedSnapshotException e) {
123      fail("fail to test snapshot manifest because region message size is too small.");
124    } finally {
125      fs.delete(regionPath, false);
126    }
127  }
128
129  private Path createDataManifest() throws IOException {
130    SnapshotDataManifest.Builder dataManifestBuilder =
131        SnapshotDataManifest.newBuilder();
132    byte[] startKey = null;
133    byte[] stopKey = null;
134    for (int i = 1; i <= TEST_NUM_REGIONS; i++) {
135      stopKey = Bytes.toBytes(String.format("%016d", i));
136      HRegionInfo regionInfo = new HRegionInfo(TABLE_NAME, startKey, stopKey, false);
137      SnapshotRegionManifest.Builder dataRegionManifestBuilder =
138          SnapshotRegionManifest.newBuilder();
139
140      for (ColumnFamilyDescriptor hcd: builder.getTableDescriptor().getColumnFamilies()) {
141        SnapshotRegionManifest.FamilyFiles.Builder family =
142            SnapshotRegionManifest.FamilyFiles.newBuilder();
143        family.setFamilyName(UnsafeByteOperations.unsafeWrap(hcd.getName()));
144        for (int j = 0; j < 100; ++j) {
145          SnapshotRegionManifest.StoreFile.Builder sfManifest =
146              SnapshotRegionManifest.StoreFile.newBuilder();
147          sfManifest.setName(String.format("%032d", i));
148          sfManifest.setFileSize((1 + i) * (1 + i) * 1024);
149          family.addStoreFiles(sfManifest.build());
150        }
151        dataRegionManifestBuilder.addFamilyFiles(family.build());
152      }
153
154      dataRegionManifestBuilder.setRegionInfo(HRegionInfo.convert(regionInfo));
155      dataManifestBuilder.addRegionManifests(dataRegionManifestBuilder.build());
156
157      startKey = stopKey;
158    }
159
160    dataManifestBuilder
161        .setTableSchema(ProtobufUtil.toTableSchema(builder.getTableDescriptor()));
162
163    SnapshotDataManifest dataManifest = dataManifestBuilder.build();
164    return writeDataManifest(dataManifest);
165  }
166
167  private Path createRegionManifest() throws IOException {
168    byte[] startKey = Bytes.toBytes("AAAAAA");
169    byte[] stopKey = Bytes.toBytes("BBBBBB");
170    HRegionInfo regionInfo = new HRegionInfo(TABLE_NAME, startKey, stopKey, false);
171    SnapshotRegionManifest.Builder dataRegionManifestBuilder = SnapshotRegionManifest.newBuilder();
172    dataRegionManifestBuilder.setRegionInfo(HRegionInfo.convert(regionInfo));
173
174    for (ColumnFamilyDescriptor hcd: builder.getTableDescriptor().getColumnFamilies()) {
175      SnapshotRegionManifest.FamilyFiles.Builder family =
176          SnapshotRegionManifest.FamilyFiles.newBuilder();
177      family.setFamilyName(UnsafeByteOperations.unsafeWrap(hcd.getName()));
178      for (int j = 0; j < TEST_NUM_REGIONFILES; ++j) {
179        SnapshotRegionManifest.StoreFile.Builder sfManifest =
180              SnapshotRegionManifest.StoreFile.newBuilder();
181        sfManifest.setName(String.format("%064d", j));
182        sfManifest.setFileSize(j * 1024);
183        family.addStoreFiles(sfManifest.build());
184      }
185      dataRegionManifestBuilder.addFamilyFiles(family.build());
186    }
187
188    SnapshotRegionManifest manifest = dataRegionManifestBuilder.build();
189    Path regionPath = new Path(snapshotDir,
190        SnapshotManifestV2.SNAPSHOT_MANIFEST_PREFIX + regionInfo.getEncodedName());
191
192    FSDataOutputStream stream = fs.create(regionPath);
193    try {
194      manifest.writeTo(stream);
195    } finally {
196      stream.close();
197    }
198
199    return regionPath;
200  }
201
202  private Path writeDataManifest(final SnapshotDataManifest manifest)
203      throws IOException {
204    Path dataRegionPath = new Path(snapshotDir, SnapshotManifest.DATA_MANIFEST_NAME);
205    FSDataOutputStream stream = fs.create(dataRegionPath);
206    try {
207      manifest.writeTo(stream);
208    } finally {
209      stream.close();
210    }
211
212    return dataRegionPath;
213  }
214}