View Javadoc

1   /**
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.Closeable;
23  import java.io.IOException;
24  import java.io.InterruptedIOException;
25  import java.lang.reflect.Proxy;
26  import java.lang.reflect.UndeclaredThrowableException;
27  import java.net.SocketTimeoutException;
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.LinkedList;
31  import java.util.List;
32  import java.util.concurrent.atomic.AtomicInteger;
33  import java.util.concurrent.atomic.AtomicReference;
34  import java.util.regex.Pattern;
35  
36  import com.google.common.annotations.VisibleForTesting;
37  
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.hadoop.conf.Configuration;
41  import org.apache.hadoop.hbase.Abortable;
42  import org.apache.hadoop.hbase.ClusterStatus;
43  import org.apache.hadoop.hbase.HBaseConfiguration;
44  import org.apache.hadoop.hbase.HColumnDescriptor;
45  import org.apache.hadoop.hbase.HConstants;
46  import org.apache.hadoop.hbase.HRegionInfo;
47  import org.apache.hadoop.hbase.HRegionLocation;
48  import org.apache.hadoop.hbase.HTableDescriptor;
49  import org.apache.hadoop.hbase.MasterNotRunningException;
50  import org.apache.hadoop.hbase.NotServingRegionException;
51  import org.apache.hadoop.hbase.RegionException;
52  import org.apache.hadoop.hbase.RemoteExceptionHandler;
53  import org.apache.hadoop.hbase.ServerName;
54  import org.apache.hadoop.hbase.TableExistsException;
55  import org.apache.hadoop.hbase.TableNotEnabledException;
56  import org.apache.hadoop.hbase.TableNotFoundException;
57  import org.apache.hadoop.hbase.UnknownRegionException;
58  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
59  import org.apache.hadoop.hbase.catalog.CatalogTracker;
60  import org.apache.hadoop.hbase.catalog.MetaReader;
61  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
62  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
63  import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
64  import org.apache.hadoop.hbase.ipc.HMasterInterface;
65  import org.apache.hadoop.hbase.ipc.HRegionInterface;
66  import org.apache.hadoop.hbase.ipc.MasterExecRPCInvoker;
67  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
68  import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest.CompactionState;
69  import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
70  import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
71  import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
72  import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
73  import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
74  import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
75  import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
76  import org.apache.hadoop.hbase.util.Addressing;
77  import org.apache.hadoop.hbase.util.Bytes;
78  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
79  import org.apache.hadoop.hbase.util.Pair;
80  import org.apache.hadoop.ipc.RemoteException;
81  import org.apache.hadoop.util.StringUtils;
82  
83  /**
84   * Provides an interface to manage HBase database table metadata + general
85   * administrative functions.  Use HBaseAdmin to create, drop, list, enable and
86   * disable tables. Use it also to add and drop table column families.
87   *
88   * <p>See {@link HTable} to add, update, and delete data from an individual table.
89   * <p>Currently HBaseAdmin instances are not expected to be long-lived.  For
90   * example, an HBaseAdmin instance will not ride over a Master restart.
91   */
92  public class HBaseAdmin implements Abortable, Closeable {
93    private final Log LOG = LogFactory.getLog(this.getClass().getName());
94  //  private final HConnection connection;
95    private HConnection connection;
96    private volatile Configuration conf;
97    private final long pause;
98    private final int numRetries;
99    // Some operations can take a long time such as disable of big table.
100   // numRetries is for 'normal' stuff... Mutliply by this factor when
101   // want to wait a long time.
102   private final int retryLongerMultiplier;
103   private boolean aborted;
104 
105   private static volatile boolean synchronousBalanceSwitchSupported = true;
106   private final boolean cleanupConnectionOnClose; // close the connection in close()
107 
108   /**
109    * Constructor
110    *
111    * @param c Configuration object
112    * @throws MasterNotRunningException if the master is not running
113    * @throws ZooKeeperConnectionException if unable to connect to zookeeper
114    */
115   public HBaseAdmin(Configuration c)
116   throws MasterNotRunningException, ZooKeeperConnectionException {
117     this.conf = HBaseConfiguration.create(c);
118     this.connection = HConnectionManager.getConnection(this.conf);
119     this.pause = this.conf.getLong("hbase.client.pause", 1000);
120     this.numRetries = this.conf.getInt("hbase.client.retries.number", 10);
121     this.retryLongerMultiplier = this.conf.getInt(
122         "hbase.client.retries.longer.multiplier", 10);
123     this.cleanupConnectionOnClose = true;
124 
125     int tries = 0;
126     while ( true ){
127       try {
128 
129         this.connection.getMaster();
130         return;
131 
132       } catch (MasterNotRunningException mnre) {
133         HConnectionManager.deleteStaleConnection(this.connection);
134         this.connection = HConnectionManager.getConnection(this.conf);
135       }
136 
137       tries++;
138       if (tries >= numRetries) {
139         // we should delete connection between client and zookeeper
140         HConnectionManager.deleteStaleConnection(this.connection);
141         throw new MasterNotRunningException("Retried " + numRetries + " times");
142       }
143 
144       try {
145         Thread.sleep(getPauseTime(tries));
146       } catch (InterruptedException e) {
147         Thread.currentThread().interrupt();
148         // we should delete connection between client and zookeeper
149         HConnectionManager.deleteStaleConnection(this.connection);
150         throw new MasterNotRunningException(
151           "Interrupted after "+tries+" tries");
152       }
153     }
154   }
155 
156  /**
157    * Constructor for externally managed HConnections.
158    * This constructor fails fast if the HMaster is not running.
159    * The HConnection can be re-used again in another attempt.
160    * This constructor fails fast.
161    *
162    * @param connection The HConnection instance to use
163    * @throws MasterNotRunningException if the master is not running
164    * @throws ZooKeeperConnectionException if unable to connect to zookeeper
165    */
166   public HBaseAdmin(HConnection connection)
167       throws MasterNotRunningException, ZooKeeperConnectionException {
168     this.conf = connection.getConfiguration();
169     this.connection = connection;
170     this.cleanupConnectionOnClose = false;
171 
172     this.pause = this.conf.getLong("hbase.client.pause", 1000);
173     this.numRetries = this.conf.getInt("hbase.client.retries.number", 10);
174     this.retryLongerMultiplier = this.conf.getInt(
175         "hbase.client.retries.longer.multiplier", 10);
176 
177     this.connection.getMaster();
178   }
179 
180   /**
181    * @return A new CatalogTracker instance; call {@link #cleanupCatalogTracker(CatalogTracker)}
182    * to cleanup the returned catalog tracker.
183    * @throws ZooKeeperConnectionException
184    * @throws IOException
185    * @see #cleanupCatalogTracker(CatalogTracker)
186    */
187   @VisibleForTesting
188   synchronized CatalogTracker getCatalogTracker()
189   throws ZooKeeperConnectionException, IOException {
190     boolean succeeded = false;
191     CatalogTracker ct = null;
192     try {
193       ct = new CatalogTracker(this.conf);
194       startCatalogTracker(ct);
195       succeeded = true;
196     } catch (InterruptedException e) {
197       // Let it out as an IOE for now until we redo all so tolerate IEs
198       Thread.currentThread().interrupt();
199       throw new IOException("Interrupted", e);
200     } finally {
201       // If we did not succeed but created a catalogtracker, clean it up. CT has a ZK instance
202       // in it and we'll leak if we don't do the 'stop'.
203       if (!succeeded && ct != null) {
204         try {
205           ct.stop();
206         } catch (RuntimeException re) {
207           LOG.error("Failed to clean up HBase's internal catalog tracker after a failed initialization. " +
208             "We may have leaked network connections to ZooKeeper; they won't be cleaned up until " +
209             "the JVM exits. If you see a large number of stale connections to ZooKeeper this is likely " +
210             "the cause. The following exception details will be needed for assistance from the " +
211             "HBase community.", re);
212         }
213         ct = null;
214       }
215     }
216     return ct;
217   }
218 
219   @VisibleForTesting
220   CatalogTracker startCatalogTracker(final CatalogTracker ct)
221   throws IOException, InterruptedException {
222     ct.start();
223     return ct;
224   }
225 
226   @VisibleForTesting
227   void cleanupCatalogTracker(final CatalogTracker ct) {
228     ct.stop();
229   }
230 
231   @Override
232   public void abort(String why, Throwable e) {
233     // Currently does nothing but throw the passed message and exception
234     this.aborted = true;
235     throw new RuntimeException(why, e);
236   }
237 
238   @Override
239   public boolean isAborted(){
240     return this.aborted;
241   }
242 
243   /** @return HConnection used by this object. */
244   public HConnection getConnection() {
245     return connection;
246   }
247 
248   /**
249    * Get a connection to the currently set master.
250    * @return proxy connection to master server for this instance
251    * @throws MasterNotRunningException if the master is not running
252    * @throws ZooKeeperConnectionException if unable to connect to zookeeper
253    * @deprecated  Master is an implementation detail for HBaseAdmin.
254    * Deprecated in HBase 0.94
255    */
256   @Deprecated
257   public HMasterInterface getMaster()
258   throws MasterNotRunningException, ZooKeeperConnectionException {
259     return this.connection.getMaster();
260   }
261 
262   /** @return - true if the master server is running
263    * @throws ZooKeeperConnectionException
264    * @throws MasterNotRunningException */
265   public boolean isMasterRunning()
266   throws MasterNotRunningException, ZooKeeperConnectionException {
267     return this.connection.isMasterRunning();
268   }
269 
270   /**
271    * @param tableName Table to check.
272    * @return True if table exists already.
273    * @throws IOException
274    */
275   public boolean tableExists(final String tableName)
276   throws IOException {
277     boolean b = false;
278     CatalogTracker ct = getCatalogTracker();
279     try {
280       b = MetaReader.tableExists(ct, tableName);
281     } finally {
282       cleanupCatalogTracker(ct);
283     }
284     return b;
285   }
286 
287   /**
288    * @param tableName Table to check.
289    * @return True if table exists already.
290    * @throws IOException
291    */
292   public boolean tableExists(final byte [] tableName)
293   throws IOException {
294     return tableExists(Bytes.toString(tableName));
295   }
296 
297   /**
298    * List all the userspace tables.  In other words, scan the META table.
299    *
300    * If we wanted this to be really fast, we could implement a special
301    * catalog table that just contains table names and their descriptors.
302    * Right now, it only exists as part of the META table's region info.
303    *
304    * @return - returns an array of HTableDescriptors
305    * @throws IOException if a remote or network exception occurs
306    */
307   public HTableDescriptor[] listTables() throws IOException {
308     return this.connection.listTables();
309   }
310 
311   /**
312    * List all the userspace tables matching the given pattern.
313    *
314    * @param pattern The compiled regular expression to match against
315    * @return - returns an array of HTableDescriptors
316    * @throws IOException if a remote or network exception occurs
317    * @see #listTables()
318    */
319   public HTableDescriptor[] listTables(Pattern pattern) throws IOException {
320     List<HTableDescriptor> matched = new LinkedList<HTableDescriptor>();
321     HTableDescriptor[] tables = listTables();
322     for (HTableDescriptor table : tables) {
323       if (pattern.matcher(table.getNameAsString()).matches()) {
324         matched.add(table);
325       }
326     }
327     return matched.toArray(new HTableDescriptor[matched.size()]);
328   }
329 
330   /**
331    * List all the userspace tables matching the given regular expression.
332    *
333    * @param regex The regular expression to match against
334    * @return - returns an array of HTableDescriptors
335    * @throws IOException if a remote or network exception occurs
336    * @see #listTables(java.util.regex.Pattern)
337    */
338   public HTableDescriptor[] listTables(String regex) throws IOException {
339     return listTables(Pattern.compile(regex));
340   }
341 
342   /**
343    * List all of the names of userspace tables.
344    * @return the list of table names
345    * @throws IOException if a remote or network exception occurs
346    */
347   public String[] getTableNames() throws IOException {
348     return this.connection.getTableNames();
349   }
350 
351   /**
352    * List all of the names of userspace tables matching the given pattern
353    * @param pattern The compiled regular expression to match against
354    * @return the list of table names
355    * @throws IOException if a remote or network exception occurs
356    */
357   public String[] getTableNames(Pattern pattern) throws IOException {
358     List<String> matched = new ArrayList<String>();
359     for (String name : this.connection.getTableNames()) {
360       if (pattern.matcher(name).matches()) {
361         matched.add(name);
362       }
363     }
364     return matched.toArray(new String[matched.size()]);
365   }
366 
367   /**
368    * List all of the names of userspace tables matching the given regex
369    * @param regex The regular expression to match against
370    * @return the list of table names
371    * @throws IOException if a remote or network exception occurs
372    */
373   public String[] getTableNames(String regex) throws IOException {
374     return getTableNames(Pattern.compile(regex));
375   }
376 
377   /**
378    * Method for getting the tableDescriptor
379    * @param tableName as a byte []
380    * @return the tableDescriptor
381    * @throws TableNotFoundException
382    * @throws IOException if a remote or network exception occurs
383    */
384   public HTableDescriptor getTableDescriptor(final byte [] tableName)
385   throws TableNotFoundException, IOException {
386     return this.connection.getHTableDescriptor(tableName);
387   }
388 
389   private long getPauseTime(int tries) {
390     int triesCount = tries;
391     if (triesCount >= HConstants.RETRY_BACKOFF.length) {
392       triesCount = HConstants.RETRY_BACKOFF.length - 1;
393     }
394     return this.pause * HConstants.RETRY_BACKOFF[triesCount];
395   }
396 
397   /**
398    * Creates a new table.
399    * Synchronous operation.
400    *
401    * @param desc table descriptor for table
402    *
403    * @throws IllegalArgumentException if the table name is reserved
404    * @throws MasterNotRunningException if master is not running
405    * @throws TableExistsException if table already exists (If concurrent
406    * threads, the table may have been created between test-for-existence
407    * and attempt-at-creation).
408    * @throws IOException if a remote or network exception occurs
409    */
410   public void createTable(HTableDescriptor desc)
411   throws IOException {
412     createTable(desc, null);
413   }
414 
415   /**
416    * Creates a new table with the specified number of regions.  The start key
417    * specified will become the end key of the first region of the table, and
418    * the end key specified will become the start key of the last region of the
419    * table (the first region has a null start key and the last region has a
420    * null end key).
421    *
422    * BigInteger math will be used to divide the key range specified into
423    * enough segments to make the required number of total regions.
424    *
425    * Synchronous operation.
426    *
427    * @param desc table descriptor for table
428    * @param startKey beginning of key range
429    * @param endKey end of key range
430    * @param numRegions the total number of regions to create
431    *
432    * @throws IllegalArgumentException if the table name is reserved
433    * @throws MasterNotRunningException if master is not running
434    * @throws TableExistsException if table already exists (If concurrent
435    * threads, the table may have been created between test-for-existence
436    * and attempt-at-creation).
437    * @throws IOException
438    */
439   public void createTable(HTableDescriptor desc, byte [] startKey,
440       byte [] endKey, int numRegions)
441   throws IOException {
442     HTableDescriptor.isLegalTableName(desc.getName());
443     if(numRegions < 3) {
444       throw new IllegalArgumentException("Must create at least three regions");
445     } else if(Bytes.compareTo(startKey, endKey) >= 0) {
446       throw new IllegalArgumentException("Start key must be smaller than end key");
447     }
448     if (numRegions == 3) {
449       createTable(desc, new byte[][] { startKey, endKey });
450       return;
451     }
452     byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
453     if(splitKeys == null || splitKeys.length != numRegions - 1) {
454       throw new IllegalArgumentException("Unable to split key range into enough regions");
455     }
456     createTable(desc, splitKeys);
457   }
458 
459   /**
460    * Creates a new table with an initial set of empty regions defined by the
461    * specified split keys.  The total number of regions created will be the
462    * number of split keys plus one. Synchronous operation.
463    * Note : Avoid passing empty split key.
464    *
465    * @param desc table descriptor for table
466    * @param splitKeys array of split keys for the initial regions of the table
467    *
468    * @throws IllegalArgumentException if the table name is reserved, if the split keys
469    * are repeated and if the split key has empty byte array.
470    * @throws MasterNotRunningException if master is not running
471    * @throws TableExistsException if table already exists (If concurrent
472    * threads, the table may have been created between test-for-existence
473    * and attempt-at-creation).
474    * @throws IOException
475    */
476   public void createTable(final HTableDescriptor desc, byte [][] splitKeys)
477   throws IOException {
478     HTableDescriptor.isLegalTableName(desc.getName());
479     try {
480       createTableAsync(desc, splitKeys);
481     } catch (SocketTimeoutException ste) {
482       LOG.warn("Creating " + desc.getNameAsString() + " took too long", ste);
483     }
484     int numRegs = splitKeys == null ? 1 : splitKeys.length + 1;
485     int prevRegCount = 0;
486     boolean doneWithMetaScan = false;
487     for (int tries = 0; tries < this.numRetries * this.retryLongerMultiplier;
488       ++tries) {
489       if (!doneWithMetaScan) {
490         // Wait for new table to come on-line
491         final AtomicInteger actualRegCount = new AtomicInteger(0);
492         MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
493           @Override
494           public boolean processRow(Result rowResult) throws IOException {
495             if (rowResult == null || rowResult.size() <= 0) {
496               return true;
497             }
498             HRegionInfo info = MetaReader.parseHRegionInfoFromCatalogResult(
499               rowResult, HConstants.REGIONINFO_QUALIFIER);
500             if (info == null) {
501               LOG.warn("No serialized HRegionInfo in " + rowResult);
502               return true;
503             }
504             if (!(Bytes.equals(info.getTableName(), desc.getName()))) {
505               return false;
506             }
507             String hostAndPort = null;
508             byte [] value = rowResult.getValue(HConstants.CATALOG_FAMILY,
509               HConstants.SERVER_QUALIFIER);
510             // Make sure that regions are assigned to server
511             if (value != null && value.length > 0) {
512               hostAndPort = Bytes.toString(value);
513             }
514             if (!(info.isOffline() || info.isSplit()) && hostAndPort != null) {
515               actualRegCount.incrementAndGet();
516             }
517             return true;
518           }
519         };
520         MetaScanner.metaScan(conf, connection, visitor, desc.getName());
521         if (actualRegCount.get() < numRegs) {
522           if (tries == this.numRetries * this.retryLongerMultiplier - 1) {
523             throw new RegionOfflineException("Only " + actualRegCount.get() +
524               " of " + numRegs + " regions are online; retries exhausted.");
525           }
526           try { // Sleep
527             Thread.sleep(getPauseTime(tries));
528           } catch (InterruptedException e) {
529             throw new InterruptedIOException("Interrupted when opening" +
530               " regions; " + actualRegCount.get() + " of " + numRegs +
531               " regions processed so far");
532           }
533           if (actualRegCount.get() > prevRegCount) { // Making progress
534             prevRegCount = actualRegCount.get();
535             tries = -1;
536           }
537         } else {
538           doneWithMetaScan = true;
539           tries = -1;
540         }
541       } else if (isTableEnabled(desc.getName())) {
542         return;
543       } else {
544         try { // Sleep
545           Thread.sleep(getPauseTime(tries));
546         } catch (InterruptedException e) {
547           throw new InterruptedIOException("Interrupted when waiting" +
548             " for table to be enabled; meta scan was done");
549         }
550       }
551     }
552     throw new TableNotEnabledException(
553       "Retries exhausted while still waiting for table: "
554       + desc.getNameAsString() + " to be enabled");
555   }
556 
557   /**
558    * Creates a new table but does not block and wait for it to come online.
559    * Asynchronous operation.  To check if the table exists, use
560    * {@link: #isTableAvailable} -- it is not safe to create an HTable
561    * instance to this table before it is available.
562    * Note : Avoid passing empty split key.
563    * @param desc table descriptor for table
564    *
565    * @throws IllegalArgumentException Bad table name, if the split keys
566    * are repeated and if the split key has empty byte array.
567    * @throws MasterNotRunningException if master is not running
568    * @throws TableExistsException if table already exists (If concurrent
569    * threads, the table may have been created between test-for-existence
570    * and attempt-at-creation).
571    * @throws IOException
572    */
573   public void createTableAsync(HTableDescriptor desc, byte [][] splitKeys)
574   throws IOException {
575     HTableDescriptor.isLegalTableName(desc.getName());
576     if(splitKeys != null && splitKeys.length > 0) {
577       Arrays.sort(splitKeys, Bytes.BYTES_COMPARATOR);
578       // Verify there are no duplicate split keys
579       byte [] lastKey = null;
580       for(byte [] splitKey : splitKeys) {
581         if (Bytes.compareTo(splitKey, HConstants.EMPTY_BYTE_ARRAY) == 0) {
582           throw new IllegalArgumentException(
583               "Empty split key must not be passed in the split keys.");
584         }
585         if(lastKey != null && Bytes.equals(splitKey, lastKey)) {
586           throw new IllegalArgumentException("All split keys must be unique, " +
587             "found duplicate: " + Bytes.toStringBinary(splitKey) +
588             ", " + Bytes.toStringBinary(lastKey));
589         }
590         lastKey = splitKey;
591       }
592     }
593     try {
594       getMaster().createTable(desc, splitKeys);
595     } catch (RemoteException e) {
596       throw e.unwrapRemoteException();
597     }
598   }
599 
600   /**
601    * Deletes a table.
602    * Synchronous operation.
603    *
604    * @param tableName name of table to delete
605    * @throws IOException if a remote or network exception occurs
606    */
607   public void deleteTable(final String tableName) throws IOException {
608     deleteTable(Bytes.toBytes(tableName));
609   }
610 
611   /**
612    * Deletes a table.
613    * Synchronous operation.
614    *
615    * @param tableName name of table to delete
616    * @throws IOException if a remote or network exception occurs
617    */
618   public void deleteTable(final byte [] tableName) throws IOException {
619     isMasterRunning();
620     HTableDescriptor.isLegalTableName(tableName);
621     HRegionLocation firstMetaServer = getFirstMetaServerForTable(tableName);
622     boolean tableExists = true;
623     try {
624       getMaster().deleteTable(tableName);
625     } catch (RemoteException e) {
626       throw RemoteExceptionHandler.decodeRemoteException(e);
627     }
628     // Wait until all regions deleted
629     HRegionInterface server =
630       connection.getHRegionConnection(firstMetaServer.getHostname(), firstMetaServer.getPort());
631     List<String> tableNameAsList = new ArrayList<String>(1);
632     tableNameAsList.add(Bytes.toString(tableName));
633     for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
634       long scannerId = -1L;
635       try {
636 
637         Scan scan = MetaReader.getScanForTableName(tableName);
638         scan.addColumn(HConstants.CATALOG_FAMILY,
639             HConstants.REGIONINFO_QUALIFIER);
640         scannerId = server.openScanner(
641           firstMetaServer.getRegionInfo().getRegionName(), scan);
642         // Get a batch at a time.
643         Result values = server.next(scannerId);
644 
645         // let us wait until .META. table is updated and
646         // HMaster removes the table from its HTableDescriptors
647         if (values == null) {
648           tableExists = false;
649           HTableDescriptor[] htds = getMaster().getHTableDescriptors(tableNameAsList);
650           tableExists = (htds != null && htds.length > 0);
651           if (!tableExists) {
652             break;
653           }
654         }
655       } catch (IOException ex) {
656         if(tries == numRetries - 1) {           // no more tries left
657           if (ex instanceof RemoteException) {
658             ex = RemoteExceptionHandler.decodeRemoteException((RemoteException) ex);
659           }
660           throw ex;
661         }
662       } finally {
663         if (scannerId != -1L) {
664           try {
665             server.close(scannerId);
666           } catch (Exception ex) {
667             LOG.warn(ex);
668           }
669         }
670       }
671       try {
672         Thread.sleep(getPauseTime(tries));
673       } catch (InterruptedException e) {
674         // continue
675       }
676     }
677 
678     if (tableExists) {
679       throw new IOException("Retries exhausted, it took too long to wait"+
680         " for the table " + Bytes.toString(tableName) + " to be deleted.");
681     }
682     // Delete cached information to prevent clients from using old locations
683     this.connection.clearRegionCache(tableName);
684     LOG.info("Deleted " + Bytes.toString(tableName));
685   }
686 
687   /**
688    * Deletes tables matching the passed in pattern and wait on completion.
689    *
690    * Warning: Use this method carefully, there is no prompting and the effect is
691    * immediate. Consider using {@link #listTables(java.lang.String)} and
692    * {@link #deleteTable(byte[])}
693    *
694    * @param regex The regular expression to match table names against
695    * @return Table descriptors for tables that couldn't be deleted
696    * @throws IOException
697    * @see #deleteTables(java.util.regex.Pattern)
698    * @see #deleteTable(java.lang.String)
699    */
700   public HTableDescriptor[] deleteTables(String regex) throws IOException {
701     return deleteTables(Pattern.compile(regex));
702   }
703 
704   /**
705    * Delete tables matching the passed in pattern and wait on completion.
706    *
707    * Warning: Use this method carefully, there is no prompting and the effect is
708    * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
709    * {@link #deleteTable(byte[])}
710    *
711    * @param pattern The pattern to match table names against
712    * @return Table descriptors for tables that couldn't be deleted
713    * @throws IOException
714    */
715   public HTableDescriptor[] deleteTables(Pattern pattern) throws IOException {
716     List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
717     for (HTableDescriptor table : listTables(pattern)) {
718       try {
719         deleteTable(table.getName());
720       } catch (IOException ex) {
721         LOG.info("Failed to delete table " + table.getNameAsString(), ex);
722         failed.add(table);
723       }
724     }
725     return failed.toArray(new HTableDescriptor[failed.size()]);
726   }
727 
728 
729   public void enableTable(final String tableName)
730   throws IOException {
731     enableTable(Bytes.toBytes(tableName));
732   }
733 
734   /**
735    * Enable a table.  May timeout.  Use {@link #enableTableAsync(byte[])}
736    * and {@link #isTableEnabled(byte[])} instead.
737    * The table has to be in disabled state for it to be enabled.
738    * @param tableName name of the table
739    * @throws IOException if a remote or network exception occurs
740    * There could be couple types of IOException
741    * TableNotFoundException means the table doesn't exist.
742    * TableNotDisabledException means the table isn't in disabled state.
743    * @see #isTableEnabled(byte[])
744    * @see #disableTable(byte[])
745    * @see #enableTableAsync(byte[])
746    */
747   public void enableTable(final byte [] tableName)
748   throws IOException {
749     enableTableAsync(tableName);
750 
751     // Wait until all regions are enabled
752     waitUntilTableIsEnabled(tableName);
753 
754     LOG.info("Enabled table " + Bytes.toString(tableName));
755   }
756 
757   /**
758    * Wait for the table to be enabled and available
759    * If enabling the table exceeds the retry period, an exception is thrown.
760    * @param tableName name of the table
761    * @throws IOException if a remote or network exception occurs or
762    *    table is not enabled after the retries period.
763    */
764   private void waitUntilTableIsEnabled(final byte[] tableName) throws IOException {
765     boolean enabled = false;
766     long start = EnvironmentEdgeManager.currentTimeMillis();
767     for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
768       enabled = isTableEnabled(tableName) && isTableAvailable(tableName);
769       if (enabled) {
770         break;
771       }
772       long sleep = getPauseTime(tries);
773       if (LOG.isDebugEnabled()) {
774         LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " +
775           "enabled in " + Bytes.toString(tableName));
776       }
777       try {
778         Thread.sleep(sleep);
779       } catch (InterruptedException e) {
780         Thread.currentThread().interrupt();
781         // Do this conversion rather than let it out because do not want to
782         // change the method signature.
783         throw new IOException("Interrupted", e);
784       }
785     }
786     if (!enabled) {
787       long msec = EnvironmentEdgeManager.currentTimeMillis() - start;
788       throw new IOException("Table '" + Bytes.toString(tableName) +
789         "' not yet enabled, after " + msec + "ms.");
790     }
791   }
792 
793   public void enableTableAsync(final String tableName)
794   throws IOException {
795     enableTableAsync(Bytes.toBytes(tableName));
796   }
797 
798   /**
799    * Brings a table on-line (enables it).  Method returns immediately though
800    * enable of table may take some time to complete, especially if the table
801    * is large (All regions are opened as part of enabling process).  Check
802    * {@link #isTableEnabled(byte[])} to learn when table is fully online.  If
803    * table is taking too long to online, check server logs.
804    * @param tableName
805    * @throws IOException
806    * @since 0.90.0
807    */
808   public void enableTableAsync(final byte [] tableName)
809   throws IOException {
810     HTableDescriptor.isLegalTableName(tableName);
811     isMasterRunning();
812     try {
813       getMaster().enableTable(tableName);
814     } catch (RemoteException e) {
815       throw e.unwrapRemoteException();
816     }
817     LOG.info("Started enable of " + Bytes.toString(tableName));
818   }
819 
820   /**
821    * Enable tables matching the passed in pattern and wait on completion.
822    *
823    * Warning: Use this method carefully, there is no prompting and the effect is
824    * immediate. Consider using {@link #listTables(java.lang.String)} and
825    * {@link #enableTable(byte[])}
826    *
827    * @param regex The regular expression to match table names against
828    * @throws IOException
829    * @see #enableTables(java.util.regex.Pattern)
830    * @see #enableTable(java.lang.String)
831    */
832   public HTableDescriptor[] enableTables(String regex) throws IOException {
833     return enableTables(Pattern.compile(regex));
834   }
835 
836   /**
837    * Enable tables matching the passed in pattern and wait on completion.
838    *
839    * Warning: Use this method carefully, there is no prompting and the effect is
840    * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
841    * {@link #enableTable(byte[])}
842    *
843    * @param pattern The pattern to match table names against
844    * @throws IOException
845    */
846   public HTableDescriptor[] enableTables(Pattern pattern) throws IOException {
847     List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
848     for (HTableDescriptor table : listTables(pattern)) {
849       if (isTableDisabled(table.getName())) {
850         try {
851           enableTable(table.getName());
852         } catch (IOException ex) {
853           LOG.info("Failed to enable table " + table.getNameAsString(), ex);
854           failed.add(table);
855         }
856       }
857     }
858     return failed.toArray(new HTableDescriptor[failed.size()]);
859   }
860 
861   public void disableTableAsync(final String tableName) throws IOException {
862     disableTableAsync(Bytes.toBytes(tableName));
863   }
864 
865   /**
866    * Starts the disable of a table.  If it is being served, the master
867    * will tell the servers to stop serving it.  This method returns immediately.
868    * The disable of a table can take some time if the table is large (all
869    * regions are closed as part of table disable operation).
870    * Call {@link #isTableDisabled(byte[])} to check for when disable completes.
871    * If table is taking too long to online, check server logs.
872    * @param tableName name of table
873    * @throws IOException if a remote or network exception occurs
874    * @see #isTableDisabled(byte[])
875    * @see #isTableEnabled(byte[])
876    * @since 0.90.0
877    */
878   public void disableTableAsync(final byte [] tableName) throws IOException {
879     HTableDescriptor.isLegalTableName(tableName);
880     isMasterRunning();
881     try {
882       getMaster().disableTable(tableName);
883     } catch (RemoteException e) {
884       throw e.unwrapRemoteException();
885     }
886     LOG.info("Started disable of " + Bytes.toString(tableName));
887   }
888 
889   public void disableTable(final String tableName)
890   throws IOException {
891     disableTable(Bytes.toBytes(tableName));
892   }
893 
894   /**
895    * Disable table and wait on completion.  May timeout eventually.  Use
896    * {@link #disableTableAsync(byte[])} and {@link #isTableDisabled(String)}
897    * instead.
898    * The table has to be in enabled state for it to be disabled.
899    * @param tableName
900    * @throws IOException
901    * There could be couple types of IOException
902    * TableNotFoundException means the table doesn't exist.
903    * TableNotEnabledException means the table isn't in enabled state.
904    */
905   public void disableTable(final byte [] tableName)
906   throws IOException {
907     disableTableAsync(tableName);
908     // Wait until table is disabled
909     boolean disabled = false;
910     for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
911       disabled = isTableDisabled(tableName);
912       if (disabled) {
913         break;
914       }
915       long sleep = getPauseTime(tries);
916       if (LOG.isDebugEnabled()) {
917         LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " +
918           "disabled in " + Bytes.toString(tableName));
919       }
920       try {
921         Thread.sleep(sleep);
922       } catch (InterruptedException e) {
923         // Do this conversion rather than let it out because do not want to
924         // change the method signature.
925         Thread.currentThread().interrupt();
926         throw new IOException("Interrupted", e);
927       }
928     }
929     if (!disabled) {
930       throw new RegionException("Retries exhausted, it took too long to wait"+
931         " for the table " + Bytes.toString(tableName) + " to be disabled.");
932     }
933     LOG.info("Disabled " + Bytes.toString(tableName));
934   }
935 
936   /**
937    * Disable tables matching the passed in pattern and wait on completion.
938    *
939    * Warning: Use this method carefully, there is no prompting and the effect is
940    * immediate. Consider using {@link #listTables(java.lang.String)} and
941    * {@link #disableTable(byte[])}
942    *
943    * @param regex The regular expression to match table names against
944    * @return Table descriptors for tables that couldn't be disabled
945    * @throws IOException
946    * @see #disableTables(java.util.regex.Pattern)
947    * @see #disableTable(java.lang.String)
948    */
949   public HTableDescriptor[] disableTables(String regex) throws IOException {
950     return disableTables(Pattern.compile(regex));
951   }
952 
953   /**
954    * Disable tables matching the passed in pattern and wait on completion.
955    *
956    * Warning: Use this method carefully, there is no prompting and the effect is
957    * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
958    * {@link #disableTable(byte[])}
959    *
960    * @param pattern The pattern to match table names against
961    * @return Table descriptors for tables that couldn't be disabled
962    * @throws IOException
963    */
964   public HTableDescriptor[] disableTables(Pattern pattern) throws IOException {
965     List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
966     for (HTableDescriptor table : listTables(pattern)) {
967       if (isTableEnabled(table.getName())) {
968         try {
969           disableTable(table.getName());
970         } catch (IOException ex) {
971           LOG.info("Failed to disable table " + table.getNameAsString(), ex);
972           failed.add(table);
973         }
974       }
975     }
976     return failed.toArray(new HTableDescriptor[failed.size()]);
977   }
978 
979   /*
980    * Checks whether table exists. If not, throws TableNotFoundException
981    * @param tableName
982    */
983   private void checkTableExistence(byte[] tableName) throws IOException {
984     if (!tableExists(tableName)) {
985       throw new TableNotFoundException(Bytes.toString(tableName));
986     }
987   }
988 
989   /**
990    * @param tableName name of table to check
991    * @return true if table is on-line
992    * @throws IOException if a remote or network exception occurs
993    */
994   public boolean isTableEnabled(String tableName) throws IOException {
995     return isTableEnabled(Bytes.toBytes(tableName));
996   }
997   /**
998    * @param tableName name of table to check
999    * @return true if table is on-line
1000    * @throws IOException if a remote or network exception occurs
1001    */
1002   public boolean isTableEnabled(byte[] tableName) throws IOException {
1003     if (!HTableDescriptor.isMetaTable(tableName)) {
1004       HTableDescriptor.isLegalTableName(tableName);
1005     }
1006     checkTableExistence(tableName);
1007     return connection.isTableEnabled(tableName);
1008   }
1009 
1010   /**
1011    * @param tableName name of table to check
1012    * @return true if table is off-line
1013    * @throws IOException if a remote or network exception occurs
1014    */
1015   public boolean isTableDisabled(final String tableName) throws IOException {
1016     return isTableDisabled(Bytes.toBytes(tableName));
1017   }
1018 
1019   /**
1020    * @param tableName name of table to check
1021    * @return true if table is off-line
1022    * @throws IOException if a remote or network exception occurs
1023    */
1024   public boolean isTableDisabled(byte[] tableName) throws IOException {
1025     if (!HTableDescriptor.isMetaTable(tableName)) {
1026       HTableDescriptor.isLegalTableName(tableName);
1027     }
1028     checkTableExistence(tableName);
1029     return connection.isTableDisabled(tableName);
1030   }
1031 
1032   /**
1033    * @param tableName name of table to check
1034    * @return true if all regions of the table are available
1035    * @throws IOException if a remote or network exception occurs
1036    */
1037   public boolean isTableAvailable(byte[] tableName) throws IOException {
1038     return connection.isTableAvailable(tableName);
1039   }
1040 
1041   /**
1042    * @param tableName name of table to check
1043    * @return true if all regions of the table are available
1044    * @throws IOException if a remote or network exception occurs
1045    */
1046   public boolean isTableAvailable(String tableName) throws IOException {
1047     return connection.isTableAvailable(Bytes.toBytes(tableName));
1048   }
1049 
1050   /**
1051    * Get the status of alter command - indicates how many regions have received
1052    * the updated schema Asynchronous operation.
1053    *
1054    * @param tableName
1055    *          name of the table to get the status of
1056    * @return Pair indicating the number of regions updated Pair.getFirst() is the
1057    *         regions that are yet to be updated Pair.getSecond() is the total number
1058    *         of regions of the table
1059    * @throws IOException
1060    *           if a remote or network exception occurs
1061    */
1062   public Pair<Integer, Integer> getAlterStatus(final byte[] tableName)
1063   throws IOException {
1064     HTableDescriptor.isLegalTableName(tableName);
1065     try {
1066       return getMaster().getAlterStatus(tableName);
1067     } catch (RemoteException e) {
1068       throw RemoteExceptionHandler.decodeRemoteException(e);
1069     }
1070   }
1071 
1072   /**
1073    * Add a column to an existing table.
1074    * Asynchronous operation.
1075    *
1076    * @param tableName name of the table to add column to
1077    * @param column column descriptor of column to be added
1078    * @throws IOException if a remote or network exception occurs
1079    */
1080   public void addColumn(final String tableName, HColumnDescriptor column)
1081   throws IOException {
1082     addColumn(Bytes.toBytes(tableName), column);
1083   }
1084 
1085   /**
1086    * Add a column to an existing table.
1087    * Asynchronous operation.
1088    *
1089    * @param tableName name of the table to add column to
1090    * @param column column descriptor of column to be added
1091    * @throws IOException if a remote or network exception occurs
1092    */
1093   public void addColumn(final byte [] tableName, HColumnDescriptor column)
1094   throws IOException {
1095     HTableDescriptor.isLegalTableName(tableName);
1096     try {
1097       getMaster().addColumn(tableName, column);
1098     } catch (RemoteException e) {
1099       throw RemoteExceptionHandler.decodeRemoteException(e);
1100     }
1101   }
1102 
1103   /**
1104    * Delete a column from a table.
1105    * Asynchronous operation.
1106    *
1107    * @param tableName name of table
1108    * @param columnName name of column to be deleted
1109    * @throws IOException if a remote or network exception occurs
1110    */
1111   public void deleteColumn(final String tableName, final String columnName)
1112   throws IOException {
1113     deleteColumn(Bytes.toBytes(tableName), Bytes.toBytes(columnName));
1114   }
1115 
1116   /**
1117    * Delete a column from a table.
1118    * Asynchronous operation.
1119    *
1120    * @param tableName name of table
1121    * @param columnName name of column to be deleted
1122    * @throws IOException if a remote or network exception occurs
1123    */
1124   public void deleteColumn(final byte [] tableName, final byte [] columnName)
1125   throws IOException {
1126     try {
1127       getMaster().deleteColumn(tableName, columnName);
1128     } catch (RemoteException e) {
1129       throw RemoteExceptionHandler.decodeRemoteException(e);
1130     }
1131   }
1132 
1133   /**
1134    * Modify an existing column family on a table.
1135    * Asynchronous operation.
1136    *
1137    * @param tableName name of table
1138    * @param descriptor new column descriptor to use
1139    * @throws IOException if a remote or network exception occurs
1140    */
1141   public void modifyColumn(final String tableName, HColumnDescriptor descriptor)
1142   throws IOException {
1143     modifyColumn(Bytes.toBytes(tableName), descriptor);
1144   }
1145 
1146   /**
1147    * Modify an existing column family on a table.
1148    * Asynchronous operation.
1149    *
1150    * @param tableName name of table
1151    * @param descriptor new column descriptor to use
1152    * @throws IOException if a remote or network exception occurs
1153    */
1154   public void modifyColumn(final byte [] tableName, HColumnDescriptor descriptor)
1155   throws IOException {
1156     try {
1157       getMaster().modifyColumn(tableName, descriptor);
1158     } catch (RemoteException re) {
1159       // Convert RE exceptions in here; client shouldn't have to deal with them,
1160       // at least w/ the type of exceptions that come out of this method:
1161       // TableNotFoundException, etc.
1162       throw RemoteExceptionHandler.decodeRemoteException(re);
1163     }
1164   }
1165 
1166   /**
1167    * Close a region. For expert-admins.  Runs close on the regionserver.  The
1168    * master will not be informed of the close.
1169    * @param regionname region name to close
1170    * @param serverName If supplied, we'll use this location rather than
1171    * the one currently in <code>.META.</code>
1172    * @throws IOException if a remote or network exception occurs
1173    */
1174   public void closeRegion(final String regionname, final String serverName)
1175   throws IOException {
1176     closeRegion(Bytes.toBytes(regionname), serverName);
1177   }
1178 
1179   /**
1180    * Close a region.  For expert-admins  Runs close on the regionserver.  The
1181    * master will not be informed of the close.
1182    * @param regionname region name to close
1183    * @param serverName The servername of the regionserver.  If passed null we
1184    * will use servername found in the .META. table. A server name
1185    * is made of host, port and startcode.  Here is an example:
1186    * <code> host187.example.com,60020,1289493121758</code>
1187    * @throws IOException if a remote or network exception occurs
1188    */
1189   public void closeRegion(final byte [] regionname, final String serverName)
1190   throws IOException {
1191     CatalogTracker ct = getCatalogTracker();
1192     try {
1193       if (serverName != null) {
1194         Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, regionname);
1195         if (pair == null || pair.getFirst() == null) {
1196           throw new UnknownRegionException(Bytes.toStringBinary(regionname));
1197         } else {
1198           closeRegion(new ServerName(serverName), pair.getFirst());
1199         }
1200       } else {
1201         Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, regionname);
1202         if (pair == null) {
1203           throw new UnknownRegionException(Bytes.toStringBinary(regionname));
1204         } else if (pair.getSecond() == null) {
1205           throw new NoServerForRegionException(Bytes.toStringBinary(regionname));
1206         } else {
1207           closeRegion(pair.getSecond(), pair.getFirst());
1208         }
1209       }
1210     } finally {
1211       cleanupCatalogTracker(ct);
1212     }
1213   }
1214 
1215   /**
1216    * For expert-admins. Runs close on the regionserver. Closes a region based on
1217    * the encoded region name. The region server name is mandatory. If the
1218    * servername is provided then based on the online regions in the specified
1219    * regionserver the specified region will be closed. The master will not be
1220    * informed of the close. Note that the regionname is the encoded regionname.
1221    *
1222    * @param encodedRegionName
1223    *          The encoded region name; i.e. the hash that makes up the region
1224    *          name suffix: e.g. if regionname is
1225    *          <code>TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396.</code>
1226    *          , then the encoded region name is:
1227    *          <code>527db22f95c8a9e0116f0cc13c680396</code>.
1228    * @param serverName
1229    *          The servername of the regionserver. A server name is made of host,
1230    *          port and startcode. This is mandatory. Here is an example:
1231    *          <code> host187.example.com,60020,1289493121758</code>
1232    * @return true if the region was closed, false if not.
1233    * @throws IOException
1234    *           if a remote or network exception occurs
1235    */
1236   public boolean closeRegionWithEncodedRegionName(final String encodedRegionName,
1237       final String serverName) throws IOException {
1238     byte[] encodedRegionNameInBytes = Bytes.toBytes(encodedRegionName);
1239     if (null == serverName || ("").equals(serverName.trim())) {
1240       throw new IllegalArgumentException(
1241           "The servername cannot be null or empty.");
1242     }
1243     ServerName sn = new ServerName(serverName);
1244     HRegionInterface rs = this.connection.getHRegionConnection(
1245         sn.getHostname(), sn.getPort());
1246     // Close the region without updating zk state.
1247     boolean isRegionClosed = rs.closeRegion(encodedRegionNameInBytes, false);
1248     if (false == isRegionClosed) {
1249       LOG.error("Not able to close the region " + encodedRegionName + ".");
1250     }
1251     return isRegionClosed;
1252   }
1253 
1254   /**
1255    * Close a region.  For expert-admins  Runs close on the regionserver.  The
1256    * master will not be informed of the close.
1257    * @param sn
1258    * @param hri
1259    * @throws IOException
1260    */
1261   public void closeRegion(final ServerName sn, final HRegionInfo hri)
1262   throws IOException {
1263     HRegionInterface rs =
1264       this.connection.getHRegionConnection(sn.getHostname(), sn.getPort());
1265     // Close the region without updating zk state.
1266     rs.closeRegion(hri, false);
1267   }
1268 
1269   /**
1270    * Flush a table or an individual region.
1271    * Synchronous operation.
1272    *
1273    * @param tableNameOrRegionName table or region to flush
1274    * @throws IOException if a remote or network exception occurs
1275    * @throws InterruptedException
1276    */
1277   public void flush(final String tableNameOrRegionName)
1278   throws IOException, InterruptedException {
1279     flush(Bytes.toBytes(tableNameOrRegionName));
1280   }
1281 
1282   /**
1283    * Flush a table or an individual region.
1284    * Synchronous operation.
1285    *
1286    * @param tableNameOrRegionName table or region to flush
1287    * @throws IOException if a remote or network exception occurs
1288    * @throws InterruptedException
1289    */
1290   public void flush(final byte [] tableNameOrRegionName)
1291   throws IOException, InterruptedException {
1292     CatalogTracker ct = getCatalogTracker();
1293     try {
1294       Pair<HRegionInfo, ServerName> regionServerPair
1295         = getRegion(tableNameOrRegionName, ct);
1296       if (regionServerPair != null) {
1297         if (regionServerPair.getSecond() == null) {
1298           throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
1299         } else {
1300           flush(regionServerPair.getSecond(), regionServerPair.getFirst());
1301         }
1302       } else {
1303         final String tableName = tableNameString(tableNameOrRegionName, ct);
1304         List<Pair<HRegionInfo, ServerName>> pairs =
1305           MetaReader.getTableRegionsAndLocations(ct,
1306               tableName);
1307         for (Pair<HRegionInfo, ServerName> pair: pairs) {
1308           if (pair.getFirst().isOffline()) continue;
1309           if (pair.getSecond() == null) continue;
1310           try {
1311             flush(pair.getSecond(), pair.getFirst());
1312           } catch (NotServingRegionException e) {
1313             if (LOG.isDebugEnabled()) {
1314               LOG.debug("Trying to flush " + pair.getFirst() + ": " +
1315                 StringUtils.stringifyException(e));
1316             }
1317           }
1318         }
1319       }
1320     } finally {
1321       cleanupCatalogTracker(ct);
1322     }
1323   }
1324 
1325   private void flush(final ServerName sn, final HRegionInfo hri)
1326   throws IOException {
1327     HRegionInterface rs =
1328       this.connection.getHRegionConnection(sn.getHostname(), sn.getPort());
1329     rs.flushRegion(hri);
1330   }
1331 
1332   /**
1333    * Compact a table or an individual region.
1334    * Asynchronous operation.
1335    *
1336    * @param tableNameOrRegionName table or region to compact
1337    * @throws IOException if a remote or network exception occurs
1338    * @throws InterruptedException
1339    */
1340   public void compact(final String tableNameOrRegionName)
1341   throws IOException, InterruptedException {
1342     compact(Bytes.toBytes(tableNameOrRegionName));
1343   }
1344 
1345   /**
1346    * Compact a table or an individual region.
1347    * Asynchronous operation.
1348    *
1349    * @param tableNameOrRegionName table or region to compact
1350    * @throws IOException if a remote or network exception occurs
1351    * @throws InterruptedException
1352    */
1353   public void compact(final byte [] tableNameOrRegionName)
1354   throws IOException, InterruptedException {
1355     compact(tableNameOrRegionName, null, false);
1356   }
1357 
1358   /**
1359    * Compact a column family within a table or region.
1360    * Asynchronous operation.
1361    *
1362    * @param tableOrRegionName table or region to compact
1363    * @param columnFamily column family within a table or region
1364    * @throws IOException if a remote or network exception occurs
1365    * @throws InterruptedException
1366    */
1367   public void compact(String tableOrRegionName, String columnFamily)
1368     throws IOException,  InterruptedException {
1369     compact(Bytes.toBytes(tableOrRegionName), Bytes.toBytes(columnFamily));
1370   }
1371 
1372   /**
1373    * Compact a column family within a table or region.
1374    * Asynchronous operation.
1375    *
1376    * @param tableNameOrRegionName table or region to compact
1377    * @param columnFamily column family within a table or region
1378    * @throws IOException if a remote or network exception occurs
1379    * @throws InterruptedException
1380    */
1381   public void compact(final byte [] tableNameOrRegionName, final byte[] columnFamily)
1382   throws IOException, InterruptedException {
1383     compact(tableNameOrRegionName, columnFamily, false);
1384   }
1385 
1386   /**
1387    * Major compact a table or an individual region.
1388    * Asynchronous operation.
1389    *
1390    * @param tableNameOrRegionName table or region to major compact
1391    * @throws IOException if a remote or network exception occurs
1392    * @throws InterruptedException
1393    */
1394   public void majorCompact(final String tableNameOrRegionName)
1395   throws IOException, InterruptedException {
1396     majorCompact(Bytes.toBytes(tableNameOrRegionName));
1397   }
1398 
1399   /**
1400    * Major compact a table or an individual region.
1401    * Asynchronous operation.
1402    *
1403    * @param tableNameOrRegionName table or region to major compact
1404    * @throws IOException if a remote or network exception occurs
1405    * @throws InterruptedException
1406    */
1407   public void majorCompact(final byte [] tableNameOrRegionName)
1408   throws IOException, InterruptedException {
1409     compact(tableNameOrRegionName, null, true);
1410   }
1411 
1412   /**
1413    * Major compact a column family within a table or region.
1414    * Asynchronous operation.
1415    *
1416    * @param tableNameOrRegionName table or region to major compact
1417    * @param columnFamily column family within a table or region
1418    * @throws IOException if a remote or network exception occurs
1419    * @throws InterruptedException
1420    */
1421   public void majorCompact(final String tableNameOrRegionName,
1422     final String columnFamily) throws IOException, InterruptedException {
1423     majorCompact(Bytes.toBytes(tableNameOrRegionName),
1424       Bytes.toBytes(columnFamily));
1425   }
1426 
1427   /**
1428    * Major compact a column family within a table or region.
1429    * Asynchronous operation.
1430    *
1431    * @param tableNameOrRegionName table or region to major compact
1432    * @param columnFamily column family within a table or region
1433    * @throws IOException if a remote or network exception occurs
1434    * @throws InterruptedException
1435    */
1436   public void majorCompact(final byte [] tableNameOrRegionName,
1437     final byte[] columnFamily) throws IOException, InterruptedException {
1438     compact(tableNameOrRegionName, columnFamily, true);
1439   }
1440 
1441   /**
1442    * Compact a table or an individual region.
1443    * Asynchronous operation.
1444    *
1445    * @param tableNameOrRegionName table or region to compact
1446    * @param columnFamily column family within a table or region
1447    * @param major True if we are to do a major compaction.
1448    * @throws IOException if a remote or network exception occurs
1449    * @throws InterruptedException
1450    */
1451   private void compact(final byte [] tableNameOrRegionName,
1452     final byte[] columnFamily, final boolean major)
1453   throws IOException, InterruptedException {
1454     CatalogTracker ct = getCatalogTracker();
1455     try {
1456       Pair<HRegionInfo, ServerName> regionServerPair
1457         = getRegion(tableNameOrRegionName, ct);
1458       if (regionServerPair != null) {
1459         if (regionServerPair.getSecond() == null) {
1460           throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
1461         } else {
1462           compact(regionServerPair.getSecond(), regionServerPair.getFirst(), major, columnFamily);
1463         }
1464       } else {
1465         final String tableName = tableNameString(tableNameOrRegionName, ct);
1466         List<Pair<HRegionInfo, ServerName>> pairs =
1467           MetaReader.getTableRegionsAndLocations(ct,
1468               tableName);
1469         for (Pair<HRegionInfo, ServerName> pair: pairs) {
1470           if (pair.getFirst().isOffline()) continue;
1471           if (pair.getSecond() == null) continue;
1472           try {
1473             compact(pair.getSecond(), pair.getFirst(), major, columnFamily);
1474           } catch (NotServingRegionException e) {
1475             if (LOG.isDebugEnabled()) {
1476               LOG.debug("Trying to" + (major ? " major" : "") + " compact " +
1477                 pair.getFirst() + ": " +
1478                 StringUtils.stringifyException(e));
1479             }
1480           }
1481         }
1482       }
1483     } finally {
1484       cleanupCatalogTracker(ct);
1485     }
1486   }
1487 
1488   private void compact(final ServerName sn, final HRegionInfo hri,
1489       final boolean major, final byte [] family)
1490   throws IOException {
1491     HRegionInterface rs =
1492       this.connection.getHRegionConnection(sn.getHostname(), sn.getPort());
1493     if (family != null) {
1494       try {
1495         rs.compactRegion(hri, major, family);
1496       } catch (IOException ioe) {
1497         String notFoundMsg = "java.lang.NoSuchMethodException: org.apache.hadoop.hbase.ipc.HRegionInterface."
1498           + "compactRegion(org.apache.hadoop.hbase.HRegionInfo, boolean, [B)";
1499         if (ioe.getMessage().contains(notFoundMsg)) {
1500           throw new IOException("per-column family compaction not supported on this version "
1501             + "of the HBase server.  You may still compact at the table or region level by "
1502           	+ "omitting the column family name.  Alternatively, you can upgrade the HBase server");
1503         }
1504         throw ioe;
1505       }
1506     } else {
1507       rs.compactRegion(hri, major);
1508     }
1509   }
1510 
1511   /**
1512    * Move the region <code>r</code> to <code>dest</code>.
1513    * @param encodedRegionName The encoded region name; i.e. the hash that makes
1514    * up the region name suffix: e.g. if regionname is
1515    * <code>TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396.</code>,
1516    * then the encoded region name is: <code>527db22f95c8a9e0116f0cc13c680396</code>.
1517    * @param destServerName The servername of the destination regionserver.  If
1518    * passed the empty byte array we'll assign to a random server.  A server name
1519    * is made of host, port and startcode.  Here is an example:
1520    * <code> host187.example.com,60020,1289493121758</code>
1521    * @throws UnknownRegionException Thrown if we can't find a region named
1522    * <code>encodedRegionName</code>
1523    * @throws ZooKeeperConnectionException
1524    * @throws MasterNotRunningException
1525    */
1526   public void move(final byte [] encodedRegionName, final byte [] destServerName)
1527   throws UnknownRegionException, MasterNotRunningException, ZooKeeperConnectionException {
1528     getMaster().move(encodedRegionName, destServerName);
1529   }
1530 
1531   /**
1532    * @param regionName
1533    *          Region name to assign.
1534    * @throws MasterNotRunningException
1535    * @throws ZooKeeperConnectionException
1536    * @throws IOException
1537    */
1538   public void assign(final byte[] regionName) throws MasterNotRunningException,
1539       ZooKeeperConnectionException, IOException {
1540     getMaster().assign(regionName);
1541   }
1542 
1543   /**
1544    * Unassign a region from current hosting regionserver.  Region will then be
1545    * assigned to a regionserver chosen at random.  Region could be reassigned
1546    * back to the same server.  Use {@link #move(byte[], byte[])} if you want
1547    * to control the region movement.
1548    * @param regionName Region to unassign. Will clear any existing RegionPlan
1549    * if one found.
1550    * @param force If true, force unassign (Will remove region from
1551    * regions-in-transition too if present. If results in double assignment
1552    * use hbck -fix to resolve. To be used by experts).
1553    * @throws MasterNotRunningException
1554    * @throws ZooKeeperConnectionException
1555    * @throws IOException
1556    */
1557   public void unassign(final byte [] regionName, final boolean force)
1558   throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
1559     getMaster().unassign(regionName, force);
1560   }
1561 
1562   /**
1563    * Turn the load balancer on or off.
1564    * @param b If true, enable balancer. If false, disable balancer.
1565    * @return Previous balancer value
1566    * @deprecated use setBalancerRunning(boolean, boolean) instead
1567    */
1568   @Deprecated
1569   public boolean balanceSwitch(final boolean b)
1570   throws MasterNotRunningException, ZooKeeperConnectionException {
1571     return getMaster().balanceSwitch(b);
1572   }
1573 
1574   /**
1575    * Turn the load balancer on or off.
1576    * @param on If true, enable balancer. If false, disable balancer.
1577    * @param synchronous If true, it waits until current balance() call, if outstanding, to return.
1578    * @return Previous balancer value
1579    */
1580   public boolean setBalancerRunning(final boolean on, final boolean synchronous)
1581   throws MasterNotRunningException, ZooKeeperConnectionException {
1582     if (synchronous && synchronousBalanceSwitchSupported) {
1583       try {
1584         return getMaster().synchronousBalanceSwitch(on);
1585       } catch (UndeclaredThrowableException ute) {
1586         String error = ute.getCause().getMessage();
1587         if (error != null && error.matches(
1588             "(?s).+NoSuchMethodException:.+synchronousBalanceSwitch.+")) {
1589           LOG.info("HMaster doesn't support synchronousBalanceSwitch");
1590           synchronousBalanceSwitchSupported = false;
1591         } else {
1592           throw ute;
1593         }
1594       }
1595     }
1596     return balanceSwitch(on);
1597   }
1598 
1599   /**
1600    * Invoke the balancer.  Will run the balancer and if regions to move, it will
1601    * go ahead and do the reassignments.  Can NOT run for various reasons.  Check
1602    * logs.
1603    * @return True if balancer ran, false otherwise.
1604    */
1605   public boolean balancer()
1606   throws MasterNotRunningException, ZooKeeperConnectionException {
1607     return getMaster().balance();
1608   }
1609 
1610   /**
1611    * Split a table or an individual region.
1612    * Asynchronous operation.
1613    *
1614    * @param tableNameOrRegionName table or region to split
1615    * @throws IOException if a remote or network exception occurs
1616    * @throws InterruptedException
1617    */
1618   public void split(final String tableNameOrRegionName)
1619   throws IOException, InterruptedException {
1620     split(Bytes.toBytes(tableNameOrRegionName));
1621   }
1622 
1623   /**
1624    * Split a table or an individual region.  Implicitly finds an optimal split
1625    * point.  Asynchronous operation.
1626    *
1627    * @param tableNameOrRegionName table to region to split
1628    * @throws IOException if a remote or network exception occurs
1629    * @throws InterruptedException
1630    */
1631   public void split(final byte [] tableNameOrRegionName)
1632   throws IOException, InterruptedException {
1633     split(tableNameOrRegionName, null);
1634   }
1635 
1636   public void split(final String tableNameOrRegionName,
1637     final String splitPoint) throws IOException, InterruptedException {
1638     split(Bytes.toBytes(tableNameOrRegionName), Bytes.toBytes(splitPoint));
1639   }
1640 
1641   /**
1642    * Split a table or an individual region.
1643    * Asynchronous operation.
1644    *
1645    * @param tableNameOrRegionName table to region to split
1646    * @param splitPoint the explicit position to split on
1647    * @throws IOException if a remote or network exception occurs
1648    * @throws InterruptedException interrupt exception occurred
1649    */
1650   public void split(final byte [] tableNameOrRegionName,
1651       final byte [] splitPoint) throws IOException, InterruptedException {
1652     CatalogTracker ct = getCatalogTracker();
1653     try {
1654       Pair<HRegionInfo, ServerName> regionServerPair
1655         = getRegion(tableNameOrRegionName, ct);
1656       if (regionServerPair != null) {
1657         if (regionServerPair.getSecond() == null) {
1658             throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
1659         } else {
1660           split(regionServerPair.getSecond(), regionServerPair.getFirst(), splitPoint);
1661         }
1662       } else {
1663         final String tableName = tableNameString(tableNameOrRegionName, ct);
1664         List<Pair<HRegionInfo, ServerName>> pairs =
1665           MetaReader.getTableRegionsAndLocations(ct,
1666               tableName);
1667         for (Pair<HRegionInfo, ServerName> pair: pairs) {
1668           // May not be a server for a particular row
1669           if (pair.getSecond() == null) continue;
1670           HRegionInfo r = pair.getFirst();
1671           // check for parents
1672           if (r.isSplitParent()) continue;
1673           // if a split point given, only split that particular region
1674           if (splitPoint != null && !r.containsRow(splitPoint)) continue;
1675           // call out to region server to do split now
1676           split(pair.getSecond(), pair.getFirst(), splitPoint);
1677         }
1678       }
1679     } finally {
1680       cleanupCatalogTracker(ct);
1681     }
1682   }
1683 
1684   private void split(final ServerName sn, final HRegionInfo hri,
1685       byte[] splitPoint) throws IOException {
1686     HRegionInterface rs =
1687       this.connection.getHRegionConnection(sn.getHostname(), sn.getPort());
1688     rs.splitRegion(hri, splitPoint);
1689   }
1690 
1691   /**
1692    * Modify an existing table, more IRB friendly version.
1693    * Asynchronous operation.  This means that it may be a while before your
1694    * schema change is updated across all of the table.
1695    *
1696    * @param tableName name of table.
1697    * @param htd modified description of the table
1698    * @throws IOException if a remote or network exception occurs
1699    */
1700   public void modifyTable(final byte [] tableName, HTableDescriptor htd)
1701   throws IOException {
1702     try {
1703       getMaster().modifyTable(tableName, htd);
1704     } catch (RemoteException re) {
1705       // Convert RE exceptions in here; client shouldn't have to deal with them,
1706       // at least w/ the type of exceptions that come out of this method:
1707       // TableNotFoundException, etc.
1708       throw RemoteExceptionHandler.decodeRemoteException(re);
1709     }
1710   }
1711 
1712   /**
1713    * @param tableNameOrRegionName Name of a table or name of a region.
1714    * @param ct A {@link CatalogTracker} instance (caller of this method usually has one).
1715    * @return a pair of HRegionInfo and ServerName if <code>tableNameOrRegionName</code> is
1716    *  a verified region name (we call {@link  MetaReader#getRegion( CatalogTracker, byte[])}
1717    *  else null.
1718    * Throw an exception if <code>tableNameOrRegionName</code> is null.
1719    * @throws IOException
1720    */
1721   Pair<HRegionInfo, ServerName> getRegion(final byte[] tableNameOrRegionName,
1722       final CatalogTracker ct) throws IOException {
1723     if (tableNameOrRegionName == null) {
1724       throw new IllegalArgumentException("Pass a table name or region name");
1725     }
1726     Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, tableNameOrRegionName);
1727     if (pair == null) {
1728       final AtomicReference<Pair<HRegionInfo, ServerName>> result =
1729         new AtomicReference<Pair<HRegionInfo, ServerName>>(null);
1730       final String encodedName = Bytes.toString(tableNameOrRegionName);
1731       MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
1732         @Override
1733         public boolean processRow(Result data) throws IOException {
1734           if (data == null || data.size() <= 0) {
1735             return true;
1736           }
1737           HRegionInfo info = MetaReader.parseHRegionInfoFromCatalogResult(
1738             data, HConstants.REGIONINFO_QUALIFIER);
1739           if (info == null) {
1740             LOG.warn("No serialized HRegionInfo in " + data);
1741             return true;
1742           }
1743           if (!encodedName.equals(info.getEncodedName())) return true;
1744           ServerName sn = MetaReader.getServerNameFromCatalogResult(data);
1745           result.set(new Pair<HRegionInfo, ServerName>(info, sn));
1746           return false; // found the region, stop
1747         }
1748       };
1749 
1750       MetaScanner.metaScan(conf, connection, visitor, null);
1751       pair = result.get();
1752     }
1753     return pair;
1754   }
1755 
1756   /**
1757    * Convert the table name byte array into a table name string and check if table
1758    * exists or not.
1759    * @param tableNameBytes Name of a table.
1760    * @param ct A {@link #CatalogTracker} instance (caller of this method usually has one).
1761    * @return tableName in string form.
1762    * @throws IOException if a remote or network exception occurs.
1763    * @throws TableNotFoundException if table does not exist.
1764    */
1765   private String tableNameString(final byte[] tableNameBytes, CatalogTracker ct)
1766       throws IOException {
1767     String tableNameString = Bytes.toString(tableNameBytes);
1768     if (!MetaReader.tableExists(ct, tableNameString)) {
1769       throw new TableNotFoundException(tableNameString);
1770     }
1771     return tableNameString;
1772   }
1773 
1774   /**
1775    * Shuts down the HBase cluster
1776    * @throws IOException if a remote or network exception occurs
1777    */
1778   public synchronized void shutdown() throws IOException {
1779     isMasterRunning();
1780     try {
1781       getMaster().shutdown();
1782     } catch (RemoteException e) {
1783       throw RemoteExceptionHandler.decodeRemoteException(e);
1784     }
1785   }
1786 
1787   /**
1788    * Shuts down the current HBase master only.
1789    * Does not shutdown the cluster.
1790    * @see #shutdown()
1791    * @throws IOException if a remote or network exception occurs
1792    */
1793   public synchronized void stopMaster() throws IOException {
1794     isMasterRunning();
1795     try {
1796       getMaster().stopMaster();
1797     } catch (RemoteException e) {
1798       throw RemoteExceptionHandler.decodeRemoteException(e);
1799     }
1800   }
1801 
1802   /**
1803    * Stop the designated regionserver
1804    * @param hostnamePort Hostname and port delimited by a <code>:</code> as in
1805    * <code>example.org:1234</code>
1806    * @throws IOException if a remote or network exception occurs
1807    */
1808   public synchronized void stopRegionServer(final String hostnamePort)
1809   throws IOException {
1810     String hostname = Addressing.parseHostname(hostnamePort);
1811     int port = Addressing.parsePort(hostnamePort);
1812     HRegionInterface rs =
1813       this.connection.getHRegionConnection(hostname, port);
1814     rs.stop("Called by admin client " + this.connection.toString());
1815   }
1816 
1817   /**
1818    * @return cluster status
1819    * @throws IOException if a remote or network exception occurs
1820    */
1821   public ClusterStatus getClusterStatus() throws IOException {
1822     return getMaster().getClusterStatus();
1823   }
1824 
1825   private HRegionLocation getFirstMetaServerForTable(final byte [] tableName)
1826   throws IOException {
1827     return connection.locateRegion(HConstants.META_TABLE_NAME,
1828       HRegionInfo.createRegionName(tableName, null, HConstants.NINES, false));
1829   }
1830 
1831   /**
1832    * @return Configuration used by the instance.
1833    */
1834   public Configuration getConfiguration() {
1835     return this.conf;
1836   }
1837 
1838   /**
1839    * Check to see if HBase is running. Throw an exception if not.
1840    *
1841    * @param conf system configuration
1842    * @throws MasterNotRunningException if the master is not running
1843    * @throws ZooKeeperConnectionException if unable to connect to zookeeper
1844    */
1845   public static void checkHBaseAvailable(Configuration conf)
1846   throws MasterNotRunningException, ZooKeeperConnectionException {
1847     Configuration copyOfConf = HBaseConfiguration.create(conf);
1848     copyOfConf.setInt("hbase.client.retries.number", 1);
1849     HBaseAdmin admin = new HBaseAdmin(copyOfConf);
1850     try {
1851       admin.close();
1852     } catch (IOException ioe) {
1853       admin.LOG.info("Failed to close connection", ioe);
1854     }
1855   }
1856 
1857   /**
1858    * get the regions of a given table.
1859    *
1860    * @param tableName the name of the table
1861    * @return Ordered list of {@link HRegionInfo}.
1862    * @throws IOException
1863    */
1864   public List<HRegionInfo> getTableRegions(final byte[] tableName)
1865   throws IOException {
1866     CatalogTracker ct = getCatalogTracker();
1867     List<HRegionInfo> Regions = null;
1868     try {
1869       Regions = MetaReader.getTableRegions(ct, tableName, true);
1870     } finally {
1871       cleanupCatalogTracker(ct);
1872     }
1873     return Regions;
1874   }
1875 
1876   public void close() throws IOException {
1877     if (cleanupConnectionOnClose && this.connection != null) {
1878       this.connection.close();
1879     }
1880   }
1881 
1882  /**
1883  * Get tableDescriptors
1884  * @param tableNames List of table names
1885  * @return HTD[] the tableDescriptor
1886  * @throws IOException if a remote or network exception occurs
1887  */
1888   public HTableDescriptor[] getTableDescriptors(List<String> tableNames)
1889   throws IOException {
1890     return this.connection.getHTableDescriptors(tableNames);
1891   }
1892 
1893   /**
1894    * Roll the log writer. That is, start writing log messages to a new file.
1895    *
1896    * @param serverName
1897    *          The servername of the regionserver. A server name is made of host,
1898    *          port and startcode. This is mandatory. Here is an example:
1899    *          <code> host187.example.com,60020,1289493121758</code>
1900    * @return If lots of logs, flush the returned regions so next time through
1901    * we can clean logs. Returns null if nothing to flush.  Names are actual
1902    * region names as returned by {@link HRegionInfo#getEncodedName()}
1903    * @throws IOException if a remote or network exception occurs
1904    * @throws FailedLogCloseException
1905    */
1906  public synchronized  byte[][] rollHLogWriter(String serverName)
1907       throws IOException, FailedLogCloseException {
1908     ServerName sn = new ServerName(serverName);
1909     HRegionInterface rs = this.connection.getHRegionConnection(
1910         sn.getHostname(), sn.getPort());
1911     return rs.rollHLogWriter();
1912   }
1913 
1914   public String[] getMasterCoprocessors() {
1915     try {
1916       return getClusterStatus().getMasterCoprocessors();
1917     } catch (IOException e) {
1918       LOG.error("Could not getClusterStatus()",e);
1919       return null;
1920     }
1921   }
1922 
1923   /**
1924    * Get the current compaction state of a table or region.
1925    * It could be in a major compaction, a minor compaction, both, or none.
1926    *
1927    * @param tableNameOrRegionName table or region to major compact
1928    * @throws IOException if a remote or network exception occurs
1929    * @throws InterruptedException
1930    * @return the current compaction state
1931    */
1932   public CompactionState getCompactionState(final String tableNameOrRegionName)
1933       throws IOException, InterruptedException {
1934     return getCompactionState(Bytes.toBytes(tableNameOrRegionName));
1935   }
1936 
1937   /**
1938    * Get the current compaction state of a table or region.
1939    * It could be in a major compaction, a minor compaction, both, or none.
1940    *
1941    * @param tableNameOrRegionName table or region to major compact
1942    * @throws IOException if a remote or network exception occurs
1943    * @throws InterruptedException
1944    * @return the current compaction state
1945    */
1946   public CompactionState getCompactionState(final byte [] tableNameOrRegionName)
1947       throws IOException, InterruptedException {
1948     CompactionState state = CompactionState.NONE;
1949     CatalogTracker ct = getCatalogTracker();
1950     try {
1951       Pair<HRegionInfo, ServerName> regionServerPair
1952         = getRegion(tableNameOrRegionName, ct);
1953       if (regionServerPair != null) {
1954         if (regionServerPair.getSecond() == null) {
1955           throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
1956         } else {
1957           ServerName sn = regionServerPair.getSecond();
1958           HRegionInterface rs =
1959             this.connection.getHRegionConnection(sn.getHostname(), sn.getPort());
1960           return CompactionState.valueOf(
1961             rs.getCompactionState(regionServerPair.getFirst().getRegionName()));
1962         }
1963       } else {
1964         final String tableName = tableNameString(tableNameOrRegionName, ct);
1965         List<Pair<HRegionInfo, ServerName>> pairs =
1966           MetaReader.getTableRegionsAndLocations(ct, tableName);
1967         for (Pair<HRegionInfo, ServerName> pair: pairs) {
1968           if (pair.getFirst().isOffline()) continue;
1969           if (pair.getSecond() == null) continue;
1970           try {
1971             ServerName sn = pair.getSecond();
1972             HRegionInterface rs =
1973               this.connection.getHRegionConnection(sn.getHostname(), sn.getPort());
1974             switch (CompactionState.valueOf(
1975               rs.getCompactionState(pair.getFirst().getRegionName()))) {
1976             case MAJOR_AND_MINOR:
1977               return CompactionState.MAJOR_AND_MINOR;
1978             case MAJOR:
1979               if (state == CompactionState.MINOR) {
1980                 return CompactionState.MAJOR_AND_MINOR;
1981               }
1982               state = CompactionState.MAJOR;
1983               break;
1984             case MINOR:
1985               if (state == CompactionState.MAJOR) {
1986                 return CompactionState.MAJOR_AND_MINOR;
1987               }
1988               state = CompactionState.MINOR;
1989               break;
1990             case NONE:
1991               default: // nothing, continue
1992             }
1993           } catch (NotServingRegionException e) {
1994             if (LOG.isDebugEnabled()) {
1995               LOG.debug("Trying to get compaction state of " +
1996                 pair.getFirst() + ": " +
1997                 StringUtils.stringifyException(e));
1998             }
1999           }
2000         }
2001       }
2002     } finally {
2003       cleanupCatalogTracker(ct);
2004     }
2005     return state;
2006   }
2007 
2008   /**
2009    * Creates and returns a proxy to the CoprocessorProtocol instance running in the
2010    * master.
2011    *
2012    * @param protocol The class or interface defining the remote protocol
2013    * @return A CoprocessorProtocol instance
2014    */
2015   public <T extends CoprocessorProtocol> T coprocessorProxy(
2016       Class<T> protocol) {
2017     return (T) Proxy.newProxyInstance(this.getClass().getClassLoader(),
2018         new Class[]{protocol},
2019         new MasterExecRPCInvoker(conf,
2020             connection,
2021             protocol));
2022   }
2023 
2024 
2025   /**
2026    * Create a timestamp consistent snapshot for the given table.
2027    * <p>
2028    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2029    * snapshot with the same name (even a different type or with different parameters) will fail with
2030    * a {@link SnapshotCreationException} indicating the duplicate naming.
2031    * <p>
2032    * Snapshot names follow the same naming constraints as tables in HBase. See
2033    * {@link HTableDescriptor#isLegalTableName(byte[])}.
2034    * @param snapshotName name of the snapshot to be created
2035    * @param tableName name of the table for which snapshot is created
2036    * @throws IOException if a remote or network exception occurs
2037    * @throws SnapshotCreationException if snapshot creation failed
2038    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2039    */
2040   public void snapshot(final String snapshotName, final String tableName) throws IOException,
2041       SnapshotCreationException, IllegalArgumentException {
2042     snapshot(snapshotName, tableName, SnapshotDescription.Type.FLUSH);
2043   }
2044 
2045   /**
2046    * Create snapshot for the given table of given flush type.
2047    * <p>
2048    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2049    * snapshot with the same name (even a different type or with different parameters) will fail with
2050    * a {@link SnapshotCreationException} indicating the duplicate naming.
2051    * <p>
2052    * Snapshot names follow the same naming constraints as tables in HBase. See
2053    * {@link HTableDescriptor#isLegalTableName(byte[])}.
2054    * @param snapshotName name of the snapshot to be created
2055    * @param tableName name of the table for which snapshot is created
2056    * @param flushType if the snapshot should be taken without flush memstore first
2057    * @throws IOException if a remote or network exception occurs
2058    * @throws SnapshotCreationException if snapshot creation failed
2059    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2060    */
2061   public void snapshot(final byte[] snapshotName, final byte[] tableName,
2062                        final SnapshotDescription.Type flushType) throws
2063       IOException, SnapshotCreationException, IllegalArgumentException {
2064       snapshot(Bytes.toString(snapshotName), Bytes.toString(tableName), flushType);
2065   }
2066 
2067   /**
2068    * Take a snapshot for the given table. If the table is enabled, a FLUSH-type snapshot will be
2069    * taken. If the table is disabled, an offline snapshot is taken.
2070    * <p>
2071    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2072    * snapshot with the same name (even a different type or with different parameters) will fail with
2073    * a {@link SnapshotCreationException} indicating the duplicate naming.
2074    * <p>
2075    * Snapshot names follow the same naming constraints as tables in HBase. See
2076    * {@link HTableDescriptor#isLegalTableName(byte[])}.
2077    * @param snapshotName name of the snapshot to be created
2078    * @param tableName name of the table for which snapshot is created
2079    * @throws IOException if a remote or network exception occurs
2080    * @throws SnapshotCreationException if snapshot creation failed
2081    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2082    */
2083   public void snapshot(final byte[] snapshotName, final byte[] tableName) throws IOException,
2084       SnapshotCreationException, IllegalArgumentException {
2085     snapshot(Bytes.toString(snapshotName), Bytes.toString(tableName));
2086   }
2087 
2088   /**
2089    * Create typed snapshot of the table.
2090    * <p>
2091    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2092    * snapshot with the same name (even a different type or with different parameters) will fail with
2093    * a {@link SnapshotCreationException} indicating the duplicate naming.
2094    * <p>
2095    * Snapshot names follow the same naming constraints as tables in HBase. See
2096    * {@link HTableDescriptor#isLegalTableName(byte[])}.
2097    * <p>
2098    * @param snapshotName name to give the snapshot on the filesystem. Must be unique from all other
2099    * snapshots stored on the cluster
2100    * @param tableName name of the table to snapshot
2101    * @param type type of snapshot to take
2102    * @throws IOException we fail to reach the master
2103    * @throws SnapshotCreationException if snapshot creation failed
2104    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2105    */
2106   public void snapshot(final String snapshotName, final String tableName,
2107       SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
2108       IllegalArgumentException {
2109     SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
2110     builder.setTable(tableName);
2111     builder.setName(snapshotName);
2112     builder.setType(type);
2113     snapshot(builder.build());
2114   }
2115 
2116   /**
2117    * Take a snapshot and wait for the server to complete that snapshot (blocking).
2118    * <p>
2119    * Only a single snapshot should be taken at a time for an instance of HBase, or results may be
2120    * undefined (you can tell multiple HBase clusters to snapshot at the same time, but only one at a
2121    * time for a single cluster).
2122    * <p>
2123    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2124    * snapshot with the same name (even a different type or with different parameters) will fail with
2125    * a {@link SnapshotCreationException} indicating the duplicate naming.
2126    * <p>
2127    * Snapshot names follow the same naming constraints as tables in HBase. See
2128    * {@link HTableDescriptor#isLegalTableName(byte[])}.
2129    * <p>
2130    * You should probably use {@link #snapshot(String, String)} or {@link #snapshot(byte[], byte[])}
2131    * unless you are sure about the type of snapshot that you want to take.
2132    * @param snapshot snapshot to take
2133    * @throws IOException or we lose contact with the master.
2134    * @throws SnapshotCreationException if snapshot failed to be taken
2135    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2136    */
2137   public void snapshot(SnapshotDescription snapshot) throws IOException, SnapshotCreationException,
2138       IllegalArgumentException {
2139     HSnapshotDescription snapshotWritable = new HSnapshotDescription(snapshot);
2140 
2141     try {
2142       // actually take the snapshot
2143       long max = takeSnapshotAsync(snapshot);
2144       long start = EnvironmentEdgeManager.currentTimeMillis();
2145       long maxPauseTime = max / this.numRetries;
2146       boolean done = false;
2147       int tries = 0;
2148       LOG.debug("Waiting a max of " + max + " ms for snapshot '" +
2149           SnapshotDescriptionUtils.toString(snapshot) + "' to complete. (max " +
2150           maxPauseTime + " ms per retry)");
2151       while (tries == 0 || (EnvironmentEdgeManager.currentTimeMillis() - start) < max && !done) {
2152         try {
2153           // sleep a backoff <= pauseTime amount
2154           long sleep = getPauseTime(tries++);
2155           sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
2156           LOG.debug("(#" + tries + ") Sleeping: " + sleep +
2157             "ms while waiting for snapshot completion.");
2158           Thread.sleep(sleep);
2159 
2160         } catch (InterruptedException e) {
2161           LOG.debug("Interrupted while waiting for snapshot " + snapshot + " to complete");
2162           Thread.currentThread().interrupt();
2163         }
2164         LOG.debug("Getting current status of snapshot from master...");
2165         done = getMaster().isSnapshotDone(snapshotWritable);
2166       }
2167 
2168       if (!done) {
2169         throw new SnapshotCreationException("Snapshot '" + snapshot.getName()
2170             + "' wasn't completed in expectedTime:" + max + " ms", snapshot);
2171       }
2172     } catch (RemoteException e) {
2173       throw RemoteExceptionHandler.decodeRemoteException(e);
2174     }
2175   }
2176 
2177   /**
2178    * Take a snapshot without waiting for the server to complete that snapshot (asynchronous)
2179    * <p>
2180    * Only a single snapshot should be taken at a time, or results may be undefined.
2181    * @param snapshot snapshot to take
2182    * @return the max time in millis to wait for the snapshot
2183    * @throws IOException if the snapshot did not succeed or we lose contact with the master.
2184    * @throws SnapshotCreationException if snapshot creation failed
2185    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2186    */
2187   public long takeSnapshotAsync(SnapshotDescription snapshot) throws IOException,
2188       SnapshotCreationException {
2189     SnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);
2190     HSnapshotDescription snapshotWritable = new HSnapshotDescription(snapshot);
2191     return getMaster().snapshot(snapshotWritable);
2192   }
2193 
2194   /**
2195    * Check the current state of the passed snapshot.
2196    * <p>
2197    * There are three possible states:
2198    * <ol>
2199    * <li>running - returns <tt>false</tt></li>
2200    * <li>finished - returns <tt>true</tt></li>
2201    * <li>finished with error - throws the exception that caused the snapshot to fail</li>
2202    * </ol>
2203    * <p>
2204    * The cluster only knows about the most recent snapshot. Therefore, if another snapshot has been
2205    * run/started since the snapshot your are checking, you will recieve an
2206    * {@link UnknownSnapshotException}.
2207    * @param snapshot description of the snapshot to check
2208    * @return <tt>true</tt> if the snapshot is completed, <tt>false</tt> if the snapshot is still
2209    * running
2210    * @throws IOException if we have a network issue
2211    * @throws HBaseSnapshotException if the snapshot failed
2212    * @throws UnknownSnapshotException if the requested snapshot is unknown
2213    */
2214   public boolean isSnapshotFinished(final SnapshotDescription snapshot)
2215       throws IOException, HBaseSnapshotException, UnknownSnapshotException {
2216     try {
2217       return getMaster().isSnapshotDone(new HSnapshotDescription(snapshot));
2218     } catch (RemoteException e) {
2219       throw RemoteExceptionHandler.decodeRemoteException(e);
2220     }
2221   }
2222 
2223   /**
2224    * Restore the specified snapshot on the original table. (The table must be disabled)
2225    * Before restoring the table, a new snapshot with the current table state is created.
2226    * In case of failure, the table will be rolled back to the its original state.
2227    *
2228    * @param snapshotName name of the snapshot to restore
2229    * @throws IOException if a remote or network exception occurs
2230    * @throws RestoreSnapshotException if snapshot failed to be restored
2231    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2232    */
2233   public void restoreSnapshot(final byte[] snapshotName)
2234       throws IOException, RestoreSnapshotException {
2235     restoreSnapshot(Bytes.toString(snapshotName));
2236   }
2237 
2238   /**
2239    * Restore the specified snapshot on the original table. (The table must be disabled)
2240    * Before restoring the table, a new snapshot with the current table state is created.
2241    * In case of failure, the table will be rolled back to its original state.
2242    *
2243    * @param snapshotName name of the snapshot to restore
2244    * @throws IOException if a remote or network exception occurs
2245    * @throws RestoreSnapshotException if snapshot failed to be restored
2246    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2247    */
2248   public void restoreSnapshot(final String snapshotName)
2249       throws IOException, RestoreSnapshotException {
2250     String rollbackSnapshot = snapshotName + "-" + EnvironmentEdgeManager.currentTimeMillis();
2251 
2252     String tableName = null;
2253     for (SnapshotDescription snapshotInfo: listSnapshots()) {
2254       if (snapshotInfo.getName().equals(snapshotName)) {
2255         tableName = snapshotInfo.getTable();
2256         break;
2257       }
2258     }
2259 
2260     if (tableName == null) {
2261       throw new RestoreSnapshotException(
2262         "Unable to find the table name for snapshot=" + snapshotName);
2263     }
2264 
2265     // Take a snapshot of the current state
2266     snapshot(rollbackSnapshot, tableName);
2267 
2268     // Restore snapshot
2269     try {
2270       internalRestoreSnapshot(snapshotName, tableName);
2271     } catch (IOException e) {
2272       // Try to rollback
2273       try {
2274         String msg = "Restore snapshot=" + snapshotName +
2275           " failed. Rollback to snapshot=" + rollbackSnapshot + " succeeded.";
2276         LOG.error(msg, e);
2277         internalRestoreSnapshot(rollbackSnapshot, tableName);
2278         throw new RestoreSnapshotException(msg, e);
2279       } catch (IOException ex) {
2280         String msg = "Failed to restore and rollback to snapshot=" + rollbackSnapshot;
2281         LOG.error(msg, ex);
2282         throw new RestoreSnapshotException(msg, ex);
2283       }
2284     }
2285   }
2286 
2287   /**
2288    * Create a new table by cloning the snapshot content.
2289    *
2290    * @param snapshotName name of the snapshot to be cloned
2291    * @param tableName name of the table where the snapshot will be restored
2292    * @throws IOException if a remote or network exception occurs
2293    * @throws TableExistsException if table to be created already exists
2294    * @throws RestoreSnapshotException if snapshot failed to be cloned
2295    * @throws IllegalArgumentException if the specified table has not a valid name
2296    */
2297   public void cloneSnapshot(final byte[] snapshotName, final byte[] tableName)
2298       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
2299     cloneSnapshot(Bytes.toString(snapshotName), Bytes.toString(tableName));
2300   }
2301 
2302   /**
2303    * Create a new table by cloning the snapshot content.
2304    *
2305    * @param snapshotName name of the snapshot to be cloned
2306    * @param tableName name of the table where the snapshot will be restored
2307    * @throws IOException if a remote or network exception occurs
2308    * @throws TableExistsException if table to be created already exists
2309    * @throws RestoreSnapshotException if snapshot failed to be cloned
2310    * @throws IllegalArgumentException if the specified table has not a valid name
2311    */
2312   public void cloneSnapshot(final String snapshotName, final String tableName)
2313       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
2314     if (tableExists(tableName)) {
2315       throw new TableExistsException(tableName);
2316     }
2317     internalRestoreSnapshot(snapshotName, tableName);
2318     waitUntilTableIsEnabled(Bytes.toBytes(tableName));
2319   }
2320 
2321   /**
2322    * Execute Restore/Clone snapshot and wait for the server to complete (blocking).
2323    * To check if the cloned table exists, use {@link #isTableAvailable} -- it is not safe to
2324    * create an HTable instance to this table before it is available.
2325    * @param snapshot snapshot to restore
2326    * @param tableName table name to restore the snapshot on
2327    * @throws IOException if a remote or network exception occurs
2328    * @throws RestoreSnapshotException if snapshot failed to be restored
2329    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2330    */
2331   private void internalRestoreSnapshot(final String snapshotName, final String tableName)
2332       throws IOException, RestoreSnapshotException {
2333     HSnapshotDescription snapshot = new HSnapshotDescription(
2334       SnapshotDescription.newBuilder().setName(snapshotName).setTable(tableName).build());
2335 
2336     try {
2337       // actually restore the snapshot
2338       getMaster().restoreSnapshot(snapshot);
2339 
2340       final long maxPauseTime = 5000;
2341       boolean done = false;
2342       int tries = 0;
2343       while (!done) {
2344         try {
2345           // sleep a backoff <= pauseTime amount
2346           long sleep = getPauseTime(tries++);
2347           sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
2348           LOG.debug(tries + ") Sleeping: " + sleep + " ms while we wait for snapshot restore to complete.");
2349           Thread.sleep(sleep);
2350         } catch (InterruptedException e) {
2351           LOG.debug("Interrupted while waiting for snapshot " + snapshot + " restore to complete");
2352           Thread.currentThread().interrupt();
2353         }
2354         LOG.debug("Getting current status of snapshot restore from master...");
2355         done = getMaster().isRestoreSnapshotDone(snapshot);
2356       }
2357       if (!done) {
2358         throw new RestoreSnapshotException("Snapshot '" + snapshot.getName() + "' wasn't restored.");
2359       }
2360     } catch (RemoteException e) {
2361       throw RemoteExceptionHandler.decodeRemoteException(e);
2362     }
2363   }
2364 
2365   /**
2366    * List completed snapshots.
2367    * @return a list of snapshot descriptors for completed snapshots
2368    * @throws IOException if a network error occurs
2369    */
2370   public List<SnapshotDescription> listSnapshots() throws IOException {
2371     List<SnapshotDescription> snapshots = new LinkedList<SnapshotDescription>();
2372     try {
2373       for (HSnapshotDescription snapshot: getMaster().getCompletedSnapshots()) {
2374         snapshots.add(snapshot.getProto());
2375       }
2376     } catch (RemoteException e) {
2377       throw RemoteExceptionHandler.decodeRemoteException(e);
2378     }
2379     return snapshots;
2380   }
2381 
2382   /**
2383    * Delete an existing snapshot.
2384    * @param snapshotName name of the snapshot
2385    * @throws IOException if a remote or network exception occurs
2386    */
2387   public void deleteSnapshot(final byte[] snapshotName) throws IOException {
2388     // make sure the snapshot is possibly valid
2389     HTableDescriptor.isLegalTableName(snapshotName, true);
2390     // do the delete
2391     SnapshotDescription snapshot = SnapshotDescription.newBuilder()
2392       .setName(Bytes.toString(snapshotName)).build();
2393     try {
2394       getMaster().deleteSnapshot(new HSnapshotDescription(snapshot));
2395     } catch (RemoteException e) {
2396       throw RemoteExceptionHandler.decodeRemoteException(e);
2397     }
2398   }
2399 
2400   /**
2401    * Delete an existing snapshot.
2402    * @param snapshotName name of the snapshot
2403    * @throws IOException if a remote or network exception occurs
2404    */
2405   public void deleteSnapshot(final String snapshotName) throws IOException {
2406     deleteSnapshot(Bytes.toBytes(snapshotName));
2407   }
2408 }