001/**
002 *
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *     http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 */
019package org.apache.hadoop.hbase.util;
020
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.fail;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.Collection;
027import java.util.EnumSet;
028import java.util.HashMap;
029import java.util.List;
030import java.util.Map;
031import java.util.Optional;
032import java.util.concurrent.CountDownLatch;
033import java.util.concurrent.ExecutorService;
034import java.util.concurrent.ScheduledThreadPoolExecutor;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.fs.FileStatus;
037import org.apache.hadoop.fs.FileSystem;
038import org.apache.hadoop.fs.Path;
039import org.apache.hadoop.hbase.ClusterMetrics;
040import org.apache.hadoop.hbase.ClusterMetrics.Option;
041import org.apache.hadoop.hbase.HBaseTestingUtility;
042import org.apache.hadoop.hbase.HColumnDescriptor;
043import org.apache.hadoop.hbase.HConstants;
044import org.apache.hadoop.hbase.HRegionLocation;
045import org.apache.hadoop.hbase.HTableDescriptor;
046import org.apache.hadoop.hbase.ServerName;
047import org.apache.hadoop.hbase.TableName;
048import org.apache.hadoop.hbase.client.Admin;
049import org.apache.hadoop.hbase.client.ClusterConnection;
050import org.apache.hadoop.hbase.client.Connection;
051import org.apache.hadoop.hbase.client.ConnectionFactory;
052import org.apache.hadoop.hbase.client.Delete;
053import org.apache.hadoop.hbase.client.Put;
054import org.apache.hadoop.hbase.client.RegionInfo;
055import org.apache.hadoop.hbase.client.RegionLocator;
056import org.apache.hadoop.hbase.client.Scan;
057import org.apache.hadoop.hbase.client.Table;
058import org.apache.hadoop.hbase.client.TableDescriptor;
059import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
060import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
061import org.apache.hadoop.hbase.coprocessor.MasterObserver;
062import org.apache.hadoop.hbase.coprocessor.ObserverContext;
063import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
064import org.apache.hadoop.hbase.master.assignment.RegionStates;
065import org.apache.hadoop.hbase.mob.MobFileName;
066import org.apache.hadoop.hbase.mob.MobUtils;
067import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
068import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
069import org.apache.zookeeper.KeeperException;
070import org.junit.rules.TestName;
071import org.slf4j.Logger;
072import org.slf4j.LoggerFactory;
073
074import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
075import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
076
077/**
078 * This is the base class for  HBaseFsck's ability to detect reasons for inconsistent tables.
079 *
080 * Actual tests are in :
081 * TestHBaseFsckTwoRS
082 * TestHBaseFsckOneRS
083 * TestHBaseFsckMOB
084 * TestHBaseFsckReplicas
085 */
086public class BaseTestHBaseFsck {
087  static final int POOL_SIZE = 7;
088  protected static final Logger LOG = LoggerFactory.getLogger(BaseTestHBaseFsck.class);
089  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
090  protected final static Configuration conf = TEST_UTIL.getConfiguration();
091  protected final static String FAM_STR = "fam";
092  protected final static byte[] FAM = Bytes.toBytes(FAM_STR);
093  protected final static int REGION_ONLINE_TIMEOUT = 800;
094  protected static AssignmentManager assignmentManager;
095  protected static RegionStates regionStates;
096  protected static ExecutorService tableExecutorService;
097  protected static ScheduledThreadPoolExecutor hbfsckExecutorService;
098  protected static ClusterConnection connection;
099  protected static Admin admin;
100
101  // for the instance, reset every test run
102  protected Table tbl;
103  protected final static byte[][] SPLITS = new byte[][] { Bytes.toBytes("A"),
104    Bytes.toBytes("B"), Bytes.toBytes("C") };
105  // one row per region.
106  protected final static byte[][] ROWKEYS= new byte[][] {
107    Bytes.toBytes("00"), Bytes.toBytes("50"), Bytes.toBytes("A0"), Bytes.toBytes("A5"),
108    Bytes.toBytes("B0"), Bytes.toBytes("B5"), Bytes.toBytes("C0"), Bytes.toBytes("C5") };
109
110  /**
111   * Debugging method to dump the contents of meta.
112   */
113  protected void dumpMeta(TableName tableName) throws IOException {
114    List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(tableName);
115    for (byte[] row : metaRows) {
116      LOG.info(Bytes.toString(row));
117    }
118  }
119
120  /**
121   * This method is used to undeploy a region -- close it and attempt to
122   * remove its state from the Master.
123   */
124  protected void undeployRegion(Connection conn, ServerName sn,
125      RegionInfo hri) throws IOException, InterruptedException {
126    try {
127      HBaseFsckRepair.closeRegionSilentlyAndWait(conn, sn, hri);
128      if (!hri.isMetaRegion()) {
129        admin.offline(hri.getRegionName());
130      }
131    } catch (IOException ioe) {
132      LOG.warn("Got exception when attempting to offline region "
133          + Bytes.toString(hri.getRegionName()), ioe);
134    }
135  }
136  /**
137   * Delete a region from assignments, meta, or completely from hdfs.
138   * @param unassign if true unassign region if assigned
139   * @param metaRow  if true remove region's row from META
140   * @param hdfs if true remove region's dir in HDFS
141   */
142  protected void deleteRegion(Configuration conf, final HTableDescriptor htd,
143      byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow,
144      boolean hdfs) throws IOException, InterruptedException {
145    deleteRegion(conf, htd, startKey, endKey, unassign, metaRow, hdfs, false,
146        RegionInfo.DEFAULT_REPLICA_ID);
147  }
148
149  /**
150   * Delete a region from assignments, meta, or completely from hdfs.
151   * @param unassign if true unassign region if assigned
152   * @param metaRow  if true remove region's row from META
153   * @param hdfs if true remove region's dir in HDFS
154   * @param regionInfoOnly if true remove a region dir's .regioninfo file
155   * @param replicaId replica id
156   */
157  protected void deleteRegion(Configuration conf, final HTableDescriptor htd,
158      byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow,
159      boolean hdfs, boolean regionInfoOnly, int replicaId)
160          throws IOException, InterruptedException {
161    LOG.info("** Before delete:");
162    dumpMeta(htd.getTableName());
163
164    List<HRegionLocation> locations;
165    try(RegionLocator rl = connection.getRegionLocator(tbl.getName())) {
166      locations = rl.getAllRegionLocations();
167    }
168
169    for (HRegionLocation location : locations) {
170      RegionInfo hri = location.getRegionInfo();
171      ServerName hsa = location.getServerName();
172      if (Bytes.compareTo(hri.getStartKey(), startKey) == 0
173          && Bytes.compareTo(hri.getEndKey(), endKey) == 0
174          && hri.getReplicaId() == replicaId) {
175
176        LOG.info("RegionName: " +hri.getRegionNameAsString());
177        byte[] deleteRow = hri.getRegionName();
178
179        if (unassign) {
180          LOG.info("Undeploying region " + hri + " from server " + hsa);
181          undeployRegion(connection, hsa, hri);
182        }
183
184        if (regionInfoOnly) {
185          LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString());
186          Path rootDir = FSUtils.getRootDir(conf);
187          FileSystem fs = rootDir.getFileSystem(conf);
188          Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),
189              hri.getEncodedName());
190          Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
191          fs.delete(hriPath, true);
192        }
193
194        if (hdfs) {
195          LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
196          Path rootDir = FSUtils.getRootDir(conf);
197          FileSystem fs = rootDir.getFileSystem(conf);
198          Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),
199              hri.getEncodedName());
200          HBaseFsck.debugLsr(conf, p);
201          boolean success = fs.delete(p, true);
202          LOG.info("Deleted " + p + " sucessfully? " + success);
203          HBaseFsck.debugLsr(conf, p);
204        }
205
206        if (metaRow) {
207          try (Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService)) {
208            Delete delete = new Delete(deleteRow);
209            meta.delete(delete);
210          }
211        }
212      }
213      LOG.info(hri.toString() + hsa.toString());
214    }
215
216    TEST_UTIL.getMetaTableRows(htd.getTableName());
217    LOG.info("*** After delete:");
218    dumpMeta(htd.getTableName());
219  }
220
221  /**
222   * Setup a clean table before we start mucking with it.
223   *
224   * It will set tbl which needs to be closed after test
225   *
226   * @throws IOException
227   * @throws InterruptedException
228   * @throws KeeperException
229   */
230  void setupTable(TableName tablename) throws Exception {
231    setupTableWithRegionReplica(tablename, 1);
232  }
233
234  /**
235   * Setup a clean table with a certain region_replica count
236   *
237   * It will set tbl which needs to be closed after test
238   *
239   * @throws Exception
240   */
241  void setupTableWithRegionReplica(TableName tablename, int replicaCount) throws Exception {
242    HTableDescriptor desc = new HTableDescriptor(tablename);
243    desc.setRegionReplication(replicaCount);
244    HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
245    desc.addFamily(hcd); // If a table has no CF's it doesn't get checked
246    createTable(TEST_UTIL, desc, SPLITS);
247
248    tbl = connection.getTable(tablename, tableExecutorService);
249    List<Put> puts = new ArrayList<>(ROWKEYS.length);
250    for (byte[] row : ROWKEYS) {
251      Put p = new Put(row);
252      p.addColumn(FAM, Bytes.toBytes("val"), row);
253      puts.add(p);
254    }
255    tbl.put(puts);
256  }
257
258  /**
259   * Setup a clean table with a mob-enabled column.
260   *
261   * @param tablename The name of a table to be created.
262   * @throws Exception
263   */
264  void setupMobTable(TableName tablename) throws Exception {
265    HTableDescriptor desc = new HTableDescriptor(tablename);
266    HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
267    hcd.setMobEnabled(true);
268    hcd.setMobThreshold(0);
269    desc.addFamily(hcd); // If a table has no CF's it doesn't get checked
270    createTable(TEST_UTIL, desc, SPLITS);
271
272    tbl = connection.getTable(tablename, tableExecutorService);
273    List<Put> puts = new ArrayList<>(ROWKEYS.length);
274    for (byte[] row : ROWKEYS) {
275      Put p = new Put(row);
276      p.addColumn(FAM, Bytes.toBytes("val"), row);
277      puts.add(p);
278    }
279    tbl.put(puts);
280  }
281
282  /**
283   * Counts the number of rows to verify data loss or non-dataloss.
284   */
285  int countRows() throws IOException {
286     return TEST_UTIL.countRows(tbl);
287  }
288
289  /**
290   * Counts the number of rows to verify data loss or non-dataloss.
291   */
292  int countRows(byte[] start, byte[] end) throws IOException {
293    return TEST_UTIL.countRows(tbl, new Scan(start, end));
294  }
295
296  /**
297   * delete table in preparation for next test
298   *
299   * @param tablename
300   * @throws IOException
301   */
302  void cleanupTable(TableName tablename) throws Exception {
303    if (tbl != null) {
304      tbl.close();
305      tbl = null;
306    }
307
308    ((ClusterConnection) connection).clearRegionLocationCache();
309    deleteTable(TEST_UTIL, tablename);
310  }
311
312  /**
313   * Get region info from local cluster.
314   */
315  Map<ServerName, List<String>> getDeployedHRIs(final Admin admin) throws IOException {
316    ClusterMetrics status = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
317    Collection<ServerName> regionServers = status.getLiveServerMetrics().keySet();
318    Map<ServerName, List<String>> mm = new HashMap<>();
319    for (ServerName hsi : regionServers) {
320      AdminProtos.AdminService.BlockingInterface server = connection.getAdmin(hsi);
321
322      // list all online regions from this region server
323      List<RegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
324      List<String> regionNames = new ArrayList<>(regions.size());
325      for (RegionInfo hri : regions) {
326        regionNames.add(hri.getRegionNameAsString());
327      }
328      mm.put(hsi, regionNames);
329    }
330    return mm;
331  }
332
333  /**
334   * Returns the HSI a region info is on.
335   */
336  ServerName findDeployedHSI(Map<ServerName, List<String>> mm, RegionInfo hri) {
337    for (Map.Entry<ServerName,List <String>> e : mm.entrySet()) {
338      if (e.getValue().contains(hri.getRegionNameAsString())) {
339        return e.getKey();
340      }
341    }
342    return null;
343  }
344
345  public void deleteTableDir(TableName table) throws IOException {
346    Path rootDir = FSUtils.getRootDir(conf);
347    FileSystem fs = rootDir.getFileSystem(conf);
348    Path p = FSUtils.getTableDir(rootDir, table);
349    HBaseFsck.debugLsr(conf, p);
350    boolean success = fs.delete(p, true);
351    LOG.info("Deleted " + p + " sucessfully? " + success);
352  }
353
354  /**
355   * We don't have an easy way to verify that a flush completed, so we loop until we find a
356   * legitimate hfile and return it.
357   * @param fs
358   * @param table
359   * @return Path of a flushed hfile.
360   * @throws IOException
361   */
362  Path getFlushedHFile(FileSystem fs, TableName table) throws IOException {
363    Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), table);
364    Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0);
365    Path famDir = new Path(regionDir, FAM_STR);
366
367    // keep doing this until we get a legit hfile
368    while (true) {
369      FileStatus[] hfFss = fs.listStatus(famDir);
370      if (hfFss.length == 0) {
371        continue;
372      }
373      for (FileStatus hfs : hfFss) {
374        if (!hfs.isDirectory()) {
375          return hfs.getPath();
376        }
377      }
378    }
379  }
380
381  /**
382   * Gets flushed mob files.
383   * @param fs The current file system.
384   * @param table The current table name.
385   * @return Path of a flushed hfile.
386   * @throws IOException
387   */
388  Path getFlushedMobFile(FileSystem fs, TableName table) throws IOException {
389    Path famDir = MobUtils.getMobFamilyPath(conf, table, FAM_STR);
390
391    // keep doing this until we get a legit hfile
392    while (true) {
393      FileStatus[] hfFss = fs.listStatus(famDir);
394      if (hfFss.length == 0) {
395        continue;
396      }
397      for (FileStatus hfs : hfFss) {
398        if (!hfs.isDirectory()) {
399          return hfs.getPath();
400        }
401      }
402    }
403  }
404
405  /**
406   * Creates a new mob file name by the old one.
407   * @param oldFileName The old mob file name.
408   * @return The new mob file name.
409   */
410  String createMobFileName(String oldFileName) {
411    MobFileName mobFileName = MobFileName.create(oldFileName);
412    String startKey = mobFileName.getStartKey();
413    String date = mobFileName.getDate();
414    return MobFileName.create(startKey, date,
415                              TEST_UTIL.getRandomUUID().toString().replaceAll("-", ""))
416      .getFileName();
417  }
418
419
420
421
422  /**
423   * Test that use this should have a timeout, because this method could potentially wait forever.
424  */
425  protected void doQuarantineTest(TableName table, HBaseFsck hbck, int check,
426                                  int corrupt, int fail, int quar, int missing) throws Exception {
427    try {
428      setupTable(table);
429      assertEquals(ROWKEYS.length, countRows());
430      admin.flush(table); // flush is async.
431
432      // Mess it up by leaving a hole in the assignment, meta, and hdfs data
433      admin.disableTable(table);
434
435      String[] args = {"-sidelineCorruptHFiles", "-repairHoles", "-ignorePreCheckPermission",
436          table.getNameAsString()};
437      HBaseFsck res = hbck.exec(hbfsckExecutorService, args);
438
439      HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
440      assertEquals(hfcc.getHFilesChecked(), check);
441      assertEquals(hfcc.getCorrupted().size(), corrupt);
442      assertEquals(hfcc.getFailures().size(), fail);
443      assertEquals(hfcc.getQuarantined().size(), quar);
444      assertEquals(hfcc.getMissing().size(), missing);
445
446      // its been fixed, verify that we can enable
447      admin.enableTableAsync(table);
448      while (!admin.isTableEnabled(table)) {
449        try {
450          Thread.sleep(250);
451        } catch (InterruptedException e) {
452          e.printStackTrace();
453          fail("Interrupted when trying to enable table " + table);
454        }
455      }
456    } finally {
457      cleanupTable(table);
458    }
459  }
460
461
462  static class MockErrorReporter implements HbckErrorReporter {
463    static int calledCount = 0;
464
465    @Override
466    public void clear() {
467      calledCount++;
468    }
469
470    @Override
471    public void report(String message) {
472      calledCount++;
473    }
474
475    @Override
476    public void reportError(String message) {
477      calledCount++;
478    }
479
480    @Override
481    public void reportError(ERROR_CODE errorCode, String message) {
482      calledCount++;
483    }
484
485    @Override
486    public void reportError(ERROR_CODE errorCode, String message, HbckTableInfo table) {
487      calledCount++;
488    }
489
490    @Override
491    public void reportError(ERROR_CODE errorCode,
492        String message, HbckTableInfo table, HbckRegionInfo info) {
493      calledCount++;
494    }
495
496    @Override
497    public void reportError(ERROR_CODE errorCode, String message,
498        HbckTableInfo table, HbckRegionInfo info1, HbckRegionInfo info2) {
499      calledCount++;
500    }
501
502    @Override
503    public int summarize() {
504      return ++calledCount;
505    }
506
507    @Override
508    public void detail(String details) {
509      calledCount++;
510    }
511
512    @Override
513    public ArrayList<ERROR_CODE> getErrorList() {
514      calledCount++;
515      return new ArrayList<>();
516    }
517
518    @Override
519    public void progress() {
520      calledCount++;
521    }
522
523    @Override
524    public void print(String message) {
525      calledCount++;
526    }
527
528    @Override
529    public void resetErrors() {
530      calledCount++;
531    }
532
533    @Override
534    public boolean tableHasErrors(HbckTableInfo table) {
535      calledCount++;
536      return false;
537    }
538  }
539
540
541  protected void deleteMetaRegion(Configuration conf, boolean unassign, boolean hdfs,
542                                  boolean regionInfoOnly) throws IOException, InterruptedException {
543    HRegionLocation metaLocation = connection.getRegionLocator(TableName.META_TABLE_NAME)
544        .getRegionLocation(HConstants.EMPTY_START_ROW);
545    ServerName hsa = metaLocation.getServerName();
546    RegionInfo hri = metaLocation.getRegionInfo();
547    if (unassign) {
548      LOG.info("Undeploying meta region " + hri + " from server " + hsa);
549      try (Connection unmanagedConnection = ConnectionFactory.createConnection(conf)) {
550        undeployRegion(unmanagedConnection, hsa, hri);
551      }
552    }
553
554    if (regionInfoOnly) {
555      LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString());
556      Path rootDir = FSUtils.getRootDir(conf);
557      FileSystem fs = rootDir.getFileSystem(conf);
558      Path p = new Path(rootDir + "/" + TableName.META_TABLE_NAME.getNameAsString(),
559          hri.getEncodedName());
560      Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
561      fs.delete(hriPath, true);
562    }
563
564    if (hdfs) {
565      LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
566      Path rootDir = FSUtils.getRootDir(conf);
567      FileSystem fs = rootDir.getFileSystem(conf);
568      Path p = new Path(rootDir + "/" + TableName.META_TABLE_NAME.getNameAsString(),
569          hri.getEncodedName());
570      HBaseFsck.debugLsr(conf, p);
571      boolean success = fs.delete(p, true);
572      LOG.info("Deleted " + p + " sucessfully? " + success);
573      HBaseFsck.debugLsr(conf, p);
574    }
575  }
576
577  @org.junit.Rule
578  public TestName name = new TestName();
579
580  public static class MasterSyncCoprocessor implements MasterCoprocessor, MasterObserver {
581    volatile CountDownLatch tableCreationLatch = null;
582    volatile CountDownLatch tableDeletionLatch = null;
583
584    @Override
585    public Optional<MasterObserver> getMasterObserver() {
586      return Optional.of(this);
587    }
588
589    @Override
590    public void postCompletedCreateTableAction(
591        final ObserverContext<MasterCoprocessorEnvironment> ctx,
592        final TableDescriptor desc,
593        final RegionInfo[] regions) throws IOException {
594      // the AccessController test, some times calls only and directly the
595      // postCompletedCreateTableAction()
596      if (tableCreationLatch != null) {
597        tableCreationLatch.countDown();
598      }
599    }
600
601    @Override
602    public void postCompletedDeleteTableAction(
603        final ObserverContext<MasterCoprocessorEnvironment> ctx,
604        final TableName tableName) throws IOException {
605      // the AccessController test, some times calls only and directly the
606      // postCompletedDeleteTableAction()
607      if (tableDeletionLatch != null) {
608        tableDeletionLatch.countDown();
609      }
610    }
611  }
612
613  public static void createTable(HBaseTestingUtility testUtil, HTableDescriptor htd,
614    byte [][] splitKeys) throws Exception {
615    // NOTE: We need a latch because admin is not sync,
616    // so the postOp coprocessor method may be called after the admin operation returned.
617    MasterSyncCoprocessor coproc = testUtil.getHBaseCluster().getMaster()
618        .getMasterCoprocessorHost().findCoprocessor(MasterSyncCoprocessor.class);
619    coproc.tableCreationLatch = new CountDownLatch(1);
620    if (splitKeys != null) {
621      admin.createTable(htd, splitKeys);
622    } else {
623      admin.createTable(htd);
624    }
625    coproc.tableCreationLatch.await();
626    coproc.tableCreationLatch = null;
627    testUtil.waitUntilAllRegionsAssigned(htd.getTableName());
628  }
629
630  public static void deleteTable(HBaseTestingUtility testUtil, TableName tableName)
631    throws Exception {
632    // NOTE: We need a latch because admin is not sync,
633    // so the postOp coprocessor method may be called after the admin operation returned.
634    MasterSyncCoprocessor coproc = testUtil.getHBaseCluster().getMaster()
635      .getMasterCoprocessorHost().findCoprocessor(MasterSyncCoprocessor.class);
636    coproc.tableDeletionLatch = new CountDownLatch(1);
637    try {
638      admin.disableTable(tableName);
639    } catch (Exception e) {
640      LOG.debug("Table: " + tableName + " already disabled, so just deleting it.");
641    }
642    admin.deleteTable(tableName);
643    coproc.tableDeletionLatch.await();
644    coproc.tableDeletionLatch = null;
645  }
646}