Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util.hbck;
019
020import static org.junit.Assert.assertEquals;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.List;
025
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.fs.FSDataOutputStream;
028import org.apache.hadoop.fs.FileSystem;
029import org.apache.hadoop.fs.Path;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.HColumnDescriptor;
032import org.apache.hadoop.hbase.HRegionLocation;
033import org.apache.hadoop.hbase.HTableDescriptor;
034import org.apache.hadoop.hbase.MetaTableAccessor;
035import org.apache.hadoop.hbase.NamespaceDescriptor;
036import org.apache.hadoop.hbase.ServerName;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.Admin;
039import org.apache.hadoop.hbase.client.Connection;
040import org.apache.hadoop.hbase.client.ConnectionFactory;
041import org.apache.hadoop.hbase.client.Delete;
042import org.apache.hadoop.hbase.client.Put;
043import org.apache.hadoop.hbase.client.RegionInfo;
044import org.apache.hadoop.hbase.client.RegionInfoBuilder;
045import org.apache.hadoop.hbase.client.RegionLocator;
046import org.apache.hadoop.hbase.client.Result;
047import org.apache.hadoop.hbase.client.ResultScanner;
048import org.apache.hadoop.hbase.client.Scan;
049import org.apache.hadoop.hbase.client.Table;
050import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
051import org.apache.hadoop.hbase.testclassification.LargeTests;
052import org.apache.hadoop.hbase.testclassification.MiscTests;
053import org.apache.hadoop.hbase.util.Bytes;
054import org.apache.hadoop.hbase.util.FSUtils;
055import org.apache.zookeeper.KeeperException;
056import org.junit.After;
057import org.junit.Before;
058import org.junit.experimental.categories.Category;
059import org.slf4j.Logger;
060import org.slf4j.LoggerFactory;
061
062/**
063 * This testing base class creates a minicluster and testing table table
064 * and shuts down the cluster afterwards. It also provides methods wipes out
065 * meta and to inject errors into meta and the file system.
066 *
067 * Tests should generally break stuff, then attempt to rebuild the meta table
068 * offline, then restart hbase, and finally perform checks.
069 *
070 * NOTE: This is a slow set of tests which takes ~30s each needs to run on a
071 * relatively beefy machine. It seems necessary to have each test in a new jvm
072 * since minicluster startup and tear downs seem to leak file handles and
073 * eventually cause out of file handle exceptions.
074 */
075@Category({MiscTests.class, LargeTests.class})
076public class OfflineMetaRebuildTestCore {
077  private final static Logger LOG = LoggerFactory
078      .getLogger(OfflineMetaRebuildTestCore.class);
079  protected HBaseTestingUtility TEST_UTIL;
080  protected Configuration conf;
081  private final static byte[] FAM = Bytes.toBytes("fam");
082
083  // for the instance, reset every test run
084  protected Table htbl;
085  protected final static byte[][] splits = new byte[][] { Bytes.toBytes("A"),
086      Bytes.toBytes("B"), Bytes.toBytes("C") };
087
088  private final static String TABLE_BASE = "tableMetaRebuild";
089  private static int tableIdx = 0;
090  protected TableName table = TableName.valueOf("tableMetaRebuild");
091  protected Connection connection;
092
093  @Before
094  public void setUpBefore() throws Exception {
095    TEST_UTIL = new HBaseTestingUtility();
096    TEST_UTIL.getConfiguration().setInt("dfs.datanode.max.xceivers", 9192);
097    TEST_UTIL.startMiniCluster(3);
098    conf = TEST_UTIL.getConfiguration();
099    this.connection = ConnectionFactory.createConnection(conf);
100    assertEquals(0, TEST_UTIL.getAdmin().listTables().length);
101
102    // setup the table
103    table = TableName.valueOf(TABLE_BASE + "-" + tableIdx);
104    tableIdx++;
105    htbl = setupTable(table);
106    populateTable(htbl);
107    assertEquals(5, scanMeta());
108    LOG.info("Table " + table + " has " + tableRowCount(conf, table)
109        + " entries.");
110    assertEquals(16, tableRowCount(conf, table));
111    TEST_UTIL.getAdmin().disableTable(table);
112    assertEquals(1, TEST_UTIL.getAdmin().listTables().length);
113  }
114
115  @After
116  public void tearDownAfter() throws Exception {
117    if (this.htbl != null) {
118      this.htbl.close();
119      this.htbl = null;
120    }
121    this.connection.close();
122    TEST_UTIL.shutdownMiniCluster();
123  }
124
125  /**
126   * Setup a clean table before we start mucking with it.
127   *
128   * @throws IOException
129   * @throws InterruptedException
130   * @throws KeeperException
131   */
132  private Table setupTable(TableName tablename) throws Exception {
133    HTableDescriptor desc = new HTableDescriptor(tablename);
134    HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
135    desc.addFamily(hcd); // If a table has no CF's it doesn't get checked
136    TEST_UTIL.getAdmin().createTable(desc, splits);
137    return this.connection.getTable(tablename);
138  }
139
140  private void dumpMeta(HTableDescriptor htd) throws IOException {
141    List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(htd.getTableName());
142    for (byte[] row : metaRows) {
143      LOG.info(Bytes.toString(row));
144    }
145  }
146
147  private void populateTable(Table tbl) throws IOException {
148    byte[] values = { 'A', 'B', 'C', 'D' };
149    List<Put> puts = new ArrayList<>();
150    for (int i = 0; i < values.length; i++) {
151      for (int j = 0; j < values.length; j++) {
152        Put put = new Put(new byte[] { values[i], values[j] });
153        put.addColumn(Bytes.toBytes("fam"), new byte[]{}, new byte[]{values[i],
154                values[j]});
155        puts.add(put);
156      }
157    }
158    tbl.put(puts);
159  }
160
161  protected void deleteRegion(Configuration conf, final Table tbl,
162      byte[] startKey, byte[] endKey) throws IOException {
163
164    LOG.info("Before delete:");
165    HTableDescriptor htd = tbl.getTableDescriptor();
166    dumpMeta(htd);
167
168    List<HRegionLocation> regions;
169    try(RegionLocator rl = connection.getRegionLocator(tbl.getName())) {
170      regions = rl.getAllRegionLocations();
171    }
172
173    for (HRegionLocation e : regions) {
174      RegionInfo hri = e.getRegionInfo();
175      ServerName hsa = e.getServerName();
176      if (Bytes.compareTo(hri.getStartKey(), startKey) == 0
177          && Bytes.compareTo(hri.getEndKey(), endKey) == 0) {
178
179        LOG.info("RegionName: " + hri.getRegionNameAsString());
180        byte[] deleteRow = hri.getRegionName();
181        TEST_UTIL.getAdmin().unassign(deleteRow, true);
182
183        LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
184        Path rootDir = FSUtils.getRootDir(conf);
185        FileSystem fs = rootDir.getFileSystem(conf);
186        Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),
187            hri.getEncodedName());
188        fs.delete(p, true);
189
190        try (Table meta = this.connection.getTable(TableName.META_TABLE_NAME)) {
191          Delete delete = new Delete(deleteRow);
192          meta.delete(delete);
193        }
194      }
195      LOG.info(hri.toString() + hsa.toString());
196    }
197
198    TEST_UTIL.getMetaTableRows(htd.getTableName());
199    LOG.info("After delete:");
200    dumpMeta(htd);
201  }
202
203  protected RegionInfo createRegion(Configuration conf, final Table htbl,
204      byte[] startKey, byte[] endKey) throws IOException {
205    Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
206    HTableDescriptor htd = htbl.getTableDescriptor();
207    RegionInfo hri = RegionInfoBuilder.newBuilder(htbl.getName())
208        .setStartKey(startKey)
209        .setEndKey(endKey)
210        .build();
211
212    LOG.info("manually adding regioninfo and hdfs data: " + hri.toString());
213    Path rootDir = FSUtils.getRootDir(conf);
214    FileSystem fs = rootDir.getFileSystem(conf);
215    Path p = new Path(FSUtils.getTableDir(rootDir, htbl.getName()),
216        hri.getEncodedName());
217    fs.mkdirs(p);
218    Path riPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
219    FSDataOutputStream out = fs.create(riPath);
220    out.write(RegionInfo.toDelimitedByteArray(hri));
221    out.close();
222
223    // add to meta.
224    MetaTableAccessor.addRegionToMeta(TEST_UTIL.getConnection(), hri);
225    meta.close();
226    return hri;
227  }
228
229  protected void wipeOutMeta() throws IOException {
230    // Mess it up by blowing up meta.
231    Admin admin = TEST_UTIL.getAdmin();
232    Scan s = new Scan();
233    Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
234    ResultScanner scanner = meta.getScanner(s);
235    List<Delete> dels = new ArrayList<>();
236    for (Result r : scanner) {
237      RegionInfo info =
238          MetaTableAccessor.getRegionInfo(r);
239      if(info != null && !info.getTable().getNamespaceAsString()
240          .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
241        Delete d = new Delete(r.getRow());
242        dels.add(d);
243        admin.unassign(r.getRow(), true);
244      }
245    }
246    meta.delete(dels);
247    scanner.close();
248    meta.close();
249  }
250
251  /**
252   * Returns the number of rows in a given table. HBase must be up and the table
253   * should be present (will wait for timeout for a while otherwise)
254   *
255   * @return # of rows in the specified table
256   */
257  protected int tableRowCount(Configuration conf, TableName table)
258      throws IOException {
259    Table t = TEST_UTIL.getConnection().getTable(table);
260    Scan st = new Scan();
261
262    ResultScanner rst = t.getScanner(st);
263    int count = 0;
264    for (@SuppressWarnings("unused")
265    Result rt : rst) {
266      count++;
267    }
268    t.close();
269    return count;
270  }
271
272  /**
273   * Dumps hbase:meta table info
274   *
275   * @return # of entries in meta.
276   */
277  protected int scanMeta() throws IOException {
278    LOG.info("Scanning META");
279    MetaTableAccessor.fullScanMetaAndPrint(TEST_UTIL.getConnection());
280    return MetaTableAccessor.fullScanRegions(TEST_UTIL.getConnection()).size();
281  }
282
283  protected HTableDescriptor[] getTables(final Configuration configuration) throws IOException {
284    HTableDescriptor[] htbls = null;
285    try (Connection connection = ConnectionFactory.createConnection(configuration)) {
286      try (Admin admin = connection.getAdmin()) {
287        htbls = admin.listTables();
288      }
289    }
290    return htbls;
291  }
292}