View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  import java.io.Closeable;
22  import java.io.IOException;
23  import java.io.InterruptedIOException;
24  import java.util.ArrayList;
25  import java.util.Arrays;
26  import java.util.HashMap;
27  import java.util.LinkedList;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Map.Entry;
31  import java.util.concurrent.ExecutionException;
32  import java.util.concurrent.Future;
33  import java.util.concurrent.TimeUnit;
34  import java.util.concurrent.TimeoutException;
35  import java.util.concurrent.atomic.AtomicInteger;
36  import java.util.concurrent.atomic.AtomicReference;
37  import java.util.regex.Pattern;
38  
39  import org.apache.commons.logging.Log;
40  import org.apache.commons.logging.LogFactory;
41  import org.apache.hadoop.conf.Configuration;
42  import org.apache.hadoop.hbase.Abortable;
43  import org.apache.hadoop.hbase.ClusterStatus;
44  import org.apache.hadoop.hbase.DoNotRetryIOException;
45  import org.apache.hadoop.hbase.HBaseConfiguration;
46  import org.apache.hadoop.hbase.HColumnDescriptor;
47  import org.apache.hadoop.hbase.HConstants;
48  import org.apache.hadoop.hbase.HRegionInfo;
49  import org.apache.hadoop.hbase.HRegionLocation;
50  import org.apache.hadoop.hbase.HTableDescriptor;
51  import org.apache.hadoop.hbase.MasterNotRunningException;
52  import org.apache.hadoop.hbase.MetaTableAccessor;
53  import org.apache.hadoop.hbase.NamespaceDescriptor;
54  import org.apache.hadoop.hbase.NotServingRegionException;
55  import org.apache.hadoop.hbase.ProcedureInfo;
56  import org.apache.hadoop.hbase.RegionLocations;
57  import org.apache.hadoop.hbase.ServerName;
58  import org.apache.hadoop.hbase.TableExistsException;
59  import org.apache.hadoop.hbase.TableName;
60  import org.apache.hadoop.hbase.TableNotDisabledException;
61  import org.apache.hadoop.hbase.TableNotFoundException;
62  import org.apache.hadoop.hbase.UnknownRegionException;
63  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
64  import org.apache.hadoop.hbase.classification.InterfaceAudience;
65  import org.apache.hadoop.hbase.classification.InterfaceStability;
66  import org.apache.hadoop.hbase.client.security.SecurityCapability;
67  import org.apache.hadoop.hbase.exceptions.DeserializationException;
68  import org.apache.hadoop.hbase.exceptions.TimeoutIOException;
69  import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
70  import org.apache.hadoop.hbase.ipc.MasterCoprocessorRpcChannel;
71  import org.apache.hadoop.hbase.ipc.RegionServerCoprocessorRpcChannel;
72  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
73  import org.apache.hadoop.hbase.protobuf.RequestConverter;
74  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
75  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
76  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
77  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest;
78  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionRequest;
79  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
80  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
81  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
82  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest;
83  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterResponse;
84  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerRequest;
85  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateConfigurationRequest;
86  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
87  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
88  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription;
89  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
90  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
91  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema;
92  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AbortProcedureRequest;
93  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AbortProcedureResponse;
94  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnRequest;
95  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnResponse;
96  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AssignRegionRequest;
97  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateNamespaceRequest;
98  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateNamespaceResponse;
99  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest;
100 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableResponse;
101 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteColumnRequest;
102 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteColumnResponse;
103 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteNamespaceRequest;
104 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteNamespaceResponse;
105 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteSnapshotRequest;
106 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteTableRequest;
107 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteTableResponse;
108 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableRequest;
109 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableResponse;
110 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsRequest;
111 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableRequest;
112 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableResponse;
113 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ExecProcedureRequest;
114 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ExecProcedureResponse;
115 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetClusterStatusRequest;
116 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetCompletedSnapshotsRequest;
117 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetNamespaceDescriptorRequest;
118 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetProcedureResultRequest;
119 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetProcedureResultResponse;
120 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusRequest;
121 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusResponse;
122 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsRequest;
123 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
124 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableNamesRequest;
125 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneRequest;
126 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneResponse;
127 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneRequest;
128 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneResponse;
129 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
130 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
131 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListNamespaceDescriptorsRequest;
132 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListProceduresRequest;
133 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableDescriptorsByNamespaceRequest;
134 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableNamesByNamespaceRequest;
135 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MajorCompactionTimestampForRegionRequest;
136 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MajorCompactionTimestampRequest;
137 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyColumnRequest;
138 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyColumnResponse;
139 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyNamespaceRequest;
140 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyNamespaceResponse;
141 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableRequest;
142 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableResponse;
143 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionRequest;
144 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotRequest;
145 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotResponse;
146 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SecurityCapabilitiesRequest;
147 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningRequest;
148 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetNormalizerRunningRequest;
149 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownRequest;
150 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotRequest;
151 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotResponse;
152 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterRequest;
153 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.TruncateTableRequest;
154 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.TruncateTableResponse;
155 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.UnassignRegionRequest;
156 import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos;
157 import org.apache.hadoop.hbase.quotas.QuotaFilter;
158 import org.apache.hadoop.hbase.quotas.QuotaRetriever;
159 import org.apache.hadoop.hbase.quotas.QuotaSettings;
160 import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
161 import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
162 import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
163 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
164 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
165 import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
166 import org.apache.hadoop.hbase.util.Addressing;
167 import org.apache.hadoop.hbase.util.Bytes;
168 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
169 import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
170 import org.apache.hadoop.hbase.util.Pair;
171 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
172 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
173 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
174 import org.apache.hadoop.ipc.RemoteException;
175 import org.apache.hadoop.util.StringUtils;
176 import org.apache.zookeeper.KeeperException;
177 
178 import com.google.common.annotations.VisibleForTesting;
179 import com.google.protobuf.ByteString;
180 import com.google.protobuf.ServiceException;
181 
182 /**
183  * HBaseAdmin is no longer a client API. It is marked InterfaceAudience.Private indicating that
184  * this is an HBase-internal class as defined in
185  * https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/InterfaceClassification.html
186  * There are no guarantees for backwards source / binary compatibility and methods or class can
187  * change or go away without deprecation.
188  * Use {@link Connection#getAdmin()} to obtain an instance of {@link Admin} instead of constructing
189  * an HBaseAdmin directly.
190  *
191  * <p>Connection should be an <i>unmanaged</i> connection obtained via
192  * {@link ConnectionFactory#createConnection(Configuration)}
193  *
194  * @see ConnectionFactory
195  * @see Connection
196  * @see Admin
197  */
198 @InterfaceAudience.Private
199 @InterfaceStability.Evolving
200 public class HBaseAdmin implements Admin {
201   private static final Log LOG = LogFactory.getLog(HBaseAdmin.class);
202 
203   private static final String ZK_IDENTIFIER_PREFIX =  "hbase-admin-on-";
204 
205   private ClusterConnection connection;
206 
207   private volatile Configuration conf;
208   private final long pause;
209   private final int numRetries;
210   // Some operations can take a long time such as disable of big table.
211   // numRetries is for 'normal' stuff... Multiply by this factor when
212   // want to wait a long time.
213   private final int retryLongerMultiplier;
214   private final int syncWaitTimeout;
215   private boolean aborted;
216   private int operationTimeout;
217 
218   private RpcRetryingCallerFactory rpcCallerFactory;
219 
220   private NonceGenerator ng;
221 
222   @Override
223   public int getOperationTimeout() {
224     return operationTimeout;
225   }
226 
227   HBaseAdmin(ClusterConnection connection) throws IOException {
228     this.conf = connection.getConfiguration();
229     this.connection = connection;
230 
231     this.pause = this.conf.getLong(HConstants.HBASE_CLIENT_PAUSE,
232         HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
233     this.numRetries = this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
234         HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
235     this.retryLongerMultiplier = this.conf.getInt(
236         "hbase.client.retries.longer.multiplier", 10);
237     this.operationTimeout = this.conf.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
238         HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT);
239     this.syncWaitTimeout = this.conf.getInt(
240       "hbase.client.sync.wait.timeout.msec", 10 * 60000); // 10min
241 
242     this.rpcCallerFactory = RpcRetryingCallerFactory.instantiate(this.conf);
243 
244     this.ng = this.connection.getNonceGenerator();
245   }
246 
247   @Override
248   public void abort(String why, Throwable e) {
249     // Currently does nothing but throw the passed message and exception
250     this.aborted = true;
251     throw new RuntimeException(why, e);
252   }
253 
254   @Override
255   public boolean isAborted(){
256     return this.aborted;
257   }
258 
259   @Override
260   public boolean abortProcedure(final long procId, final boolean mayInterruptIfRunning)
261   throws IOException {
262     return get(abortProcedureAsync(procId, mayInterruptIfRunning), this.syncWaitTimeout,
263       TimeUnit.MILLISECONDS);
264   }
265 
266   @Override
267   public Future<Boolean> abortProcedureAsync(
268     final long procId,
269     final boolean mayInterruptIfRunning) throws IOException {
270     Boolean abortProcResponse = executeCallable(
271       new MasterCallable<AbortProcedureResponse>(getConnection()) {
272     @Override
273     public AbortProcedureResponse call(int callTimeout) throws ServiceException {
274       AbortProcedureRequest abortProcRequest =
275           AbortProcedureRequest.newBuilder().setProcId(procId).build();
276       return master.abortProcedure(null,abortProcRequest);
277       }
278     }).getIsProcedureAborted();
279 
280     AbortProcedureFuture abortProcFuture =
281         new AbortProcedureFuture(this, procId, abortProcResponse);
282     return abortProcFuture;
283   }
284 
285   private static class AbortProcedureFuture extends ProcedureFuture<Boolean> {
286     private boolean isAbortInProgress;
287 
288     public AbortProcedureFuture(
289         final HBaseAdmin admin,
290         final Long procId,
291         final Boolean abortProcResponse) {
292       super(admin, procId);
293       this.isAbortInProgress = abortProcResponse;
294     }
295 
296     @Override
297     public Boolean get(long timeout, TimeUnit unit)
298         throws InterruptedException, ExecutionException, TimeoutException {
299       if (!this.isAbortInProgress) {
300         return false;
301       }
302       super.get(timeout, unit);
303       return true;
304     }
305   }
306 
307   /** @return HConnection used by this object. */
308   @Override
309   public HConnection getConnection() {
310     return connection;
311   }
312 
313   @Override
314   public boolean tableExists(final TableName tableName) throws IOException {
315     return executeCallable(new ConnectionCallable<Boolean>(getConnection()) {
316       @Override
317       public Boolean call(int callTimeout) throws ServiceException, IOException {
318         return MetaTableAccessor.tableExists(connection, tableName);
319       }
320     });
321   }
322 
323   @Override
324   public HTableDescriptor[] listTables() throws IOException {
325     return listTables((Pattern)null, false);
326   }
327 
328   @Override
329   public HTableDescriptor[] listTables(Pattern pattern) throws IOException {
330     return listTables(pattern, false);
331   }
332 
333   @Override
334   public HTableDescriptor[] listTables(String regex) throws IOException {
335     return listTables(Pattern.compile(regex), false);
336   }
337 
338   @Override
339   public HTableDescriptor[] listTables(final Pattern pattern, final boolean includeSysTables)
340       throws IOException {
341     return executeCallable(new MasterCallable<HTableDescriptor[]>(getConnection()) {
342       @Override
343       public HTableDescriptor[] call(int callTimeout) throws ServiceException {
344         GetTableDescriptorsRequest req =
345             RequestConverter.buildGetTableDescriptorsRequest(pattern, includeSysTables);
346         return ProtobufUtil.getHTableDescriptorArray(master.getTableDescriptors(null, req));
347       }
348     });
349   }
350 
351   @Override
352   public HTableDescriptor[] listTables(String regex, boolean includeSysTables)
353       throws IOException {
354     return listTables(Pattern.compile(regex), includeSysTables);
355   }
356 
357   @Override
358   public TableName[] listTableNames() throws IOException {
359     return listTableNames((Pattern)null, false);
360   }
361 
362   @Override
363   public TableName[] listTableNames(Pattern pattern) throws IOException {
364     return listTableNames(pattern, false);
365   }
366 
367   @Override
368   public TableName[] listTableNames(String regex) throws IOException {
369     return listTableNames(Pattern.compile(regex), false);
370   }
371 
372   @Override
373   public TableName[] listTableNames(final Pattern pattern, final boolean includeSysTables)
374       throws IOException {
375     return executeCallable(new MasterCallable<TableName[]>(getConnection()) {
376       @Override
377       public TableName[] call(int callTimeout) throws ServiceException {
378         GetTableNamesRequest req =
379             RequestConverter.buildGetTableNamesRequest(pattern, includeSysTables);
380         return ProtobufUtil.getTableNameArray(master.getTableNames(null, req)
381             .getTableNamesList());
382       }
383     });
384   }
385 
386   @Override
387   public TableName[] listTableNames(final String regex, final boolean includeSysTables)
388       throws IOException {
389     return listTableNames(Pattern.compile(regex), includeSysTables);
390   }
391 
392   @Override
393   public HTableDescriptor getTableDescriptor(final TableName tableName) throws IOException {
394      return getTableDescriptor(tableName, getConnection(), rpcCallerFactory, operationTimeout);
395   }
396 
397   static HTableDescriptor getTableDescriptor(final TableName tableName, HConnection connection,
398       RpcRetryingCallerFactory rpcCallerFactory, int operationTimeout) throws IOException {
399       if (tableName == null) return null;
400       HTableDescriptor htd = executeCallable(new MasterCallable<HTableDescriptor>(connection) {
401         @Override
402         public HTableDescriptor call(int callTimeout) throws ServiceException {
403           GetTableDescriptorsResponse htds;
404           GetTableDescriptorsRequest req =
405                   RequestConverter.buildGetTableDescriptorsRequest(tableName);
406           htds = master.getTableDescriptors(null, req);
407 
408           if (!htds.getTableSchemaList().isEmpty()) {
409             return HTableDescriptor.convert(htds.getTableSchemaList().get(0));
410           }
411           return null;
412         }
413       }, rpcCallerFactory, operationTimeout);
414       if (htd != null) {
415         return htd;
416       }
417       throw new TableNotFoundException(tableName.getNameAsString());
418   }
419 
420   private long getPauseTime(int tries) {
421     int triesCount = tries;
422     if (triesCount >= HConstants.RETRY_BACKOFF.length) {
423       triesCount = HConstants.RETRY_BACKOFF.length - 1;
424     }
425     return this.pause * HConstants.RETRY_BACKOFF[triesCount];
426   }
427 
428   @Override
429   public void createTable(HTableDescriptor desc)
430   throws IOException {
431     createTable(desc, null);
432   }
433 
434   @Override
435   public void createTable(HTableDescriptor desc, byte [] startKey,
436       byte [] endKey, int numRegions)
437   throws IOException {
438     if(numRegions < 3) {
439       throw new IllegalArgumentException("Must create at least three regions");
440     } else if(Bytes.compareTo(startKey, endKey) >= 0) {
441       throw new IllegalArgumentException("Start key must be smaller than end key");
442     }
443     if (numRegions == 3) {
444       createTable(desc, new byte[][]{startKey, endKey});
445       return;
446     }
447     byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
448     if(splitKeys == null || splitKeys.length != numRegions - 1) {
449       throw new IllegalArgumentException("Unable to split key range into enough regions");
450     }
451     createTable(desc, splitKeys);
452   }
453 
454   @Override
455   public void createTable(final HTableDescriptor desc, byte [][] splitKeys)
456       throws IOException {
457     get(createTableAsync(desc, splitKeys), syncWaitTimeout, TimeUnit.MILLISECONDS);
458   }
459 
460   @Override
461   public Future<Void> createTableAsync(final HTableDescriptor desc, final byte[][] splitKeys)
462       throws IOException {
463     if (desc.getTableName() == null) {
464       throw new IllegalArgumentException("TableName cannot be null");
465     }
466     if (splitKeys != null && splitKeys.length > 0) {
467       Arrays.sort(splitKeys, Bytes.BYTES_COMPARATOR);
468       // Verify there are no duplicate split keys
469       byte[] lastKey = null;
470       for (byte[] splitKey : splitKeys) {
471         if (Bytes.compareTo(splitKey, HConstants.EMPTY_BYTE_ARRAY) == 0) {
472           throw new IllegalArgumentException(
473               "Empty split key must not be passed in the split keys.");
474         }
475         if (lastKey != null && Bytes.equals(splitKey, lastKey)) {
476           throw new IllegalArgumentException("All split keys must be unique, " +
477             "found duplicate: " + Bytes.toStringBinary(splitKey) +
478             ", " + Bytes.toStringBinary(lastKey));
479         }
480         lastKey = splitKey;
481       }
482     }
483 
484     CreateTableResponse response = executeCallable(
485         new MasterCallable<CreateTableResponse>(getConnection()) {
486       @Override
487       public CreateTableResponse call(int callTimeout) throws ServiceException {
488         CreateTableRequest request = RequestConverter.buildCreateTableRequest(
489           desc, splitKeys, ng.getNonceGroup(), ng.newNonce());
490         return master.createTable(null, request);
491       }
492     });
493     return new CreateTableFuture(this, desc, splitKeys, response);
494   }
495 
496   private static class CreateTableFuture extends TableFuture<Void> {
497     private final HTableDescriptor desc;
498     private final byte[][] splitKeys;
499 
500     public CreateTableFuture(final HBaseAdmin admin, final HTableDescriptor desc,
501         final byte[][] splitKeys, final CreateTableResponse response) {
502       super(admin, desc.getTableName(),
503               (response != null && response.hasProcId()) ? response.getProcId() : null);
504       this.splitKeys = splitKeys;
505       this.desc = desc;
506     }
507 
508     @Override
509     protected HTableDescriptor getTableDescriptor() {
510       return desc;
511     }
512 
513     @Override
514     public String getOperationType() {
515       return "CREATE";
516     }
517 
518     @Override
519     protected Void waitOperationResult(final long deadlineTs) throws IOException, TimeoutException {
520       waitForTableEnabled(deadlineTs);
521       waitForAllRegionsOnline(deadlineTs, splitKeys);
522       return null;
523     }
524   }
525 
526   @Override
527   public void deleteTable(final TableName tableName) throws IOException {
528     get(deleteTableAsync(tableName), syncWaitTimeout, TimeUnit.MILLISECONDS);
529   }
530 
531   @Override
532   public Future<Void> deleteTableAsync(final TableName tableName) throws IOException {
533     DeleteTableResponse response = executeCallable(
534         new MasterCallable<DeleteTableResponse>(getConnection()) {
535       @Override
536       public DeleteTableResponse call(int callTimeout) throws ServiceException {
537         DeleteTableRequest req =
538             RequestConverter.buildDeleteTableRequest(tableName, ng.getNonceGroup(), ng.newNonce());
539         return master.deleteTable(null,req);
540       }
541     });
542     return new DeleteTableFuture(this, tableName, response);
543   }
544 
545   private static class DeleteTableFuture extends TableFuture<Void> {
546     public DeleteTableFuture(final HBaseAdmin admin, final TableName tableName,
547         final DeleteTableResponse response) {
548       super(admin, tableName,
549               (response != null && response.hasProcId()) ? response.getProcId() : null);
550     }
551 
552     @Override
553     public String getOperationType() {
554       return "DELETE";
555     }
556 
557     @Override
558     protected Void waitOperationResult(final long deadlineTs)
559         throws IOException, TimeoutException {
560       waitTableNotFound(deadlineTs);
561       return null;
562     }
563 
564     @Override
565     protected Void postOperationResult(final Void result, final long deadlineTs)
566         throws IOException, TimeoutException {
567       // Delete cached information to prevent clients from using old locations
568       getAdmin().getConnection().clearRegionCache(getTableName());
569       return super.postOperationResult(result, deadlineTs);
570     }
571   }
572 
573   @Override
574   public HTableDescriptor[] deleteTables(String regex) throws IOException {
575     return deleteTables(Pattern.compile(regex));
576   }
577 
578   /**
579    * Delete tables matching the passed in pattern and wait on completion.
580    *
581    * Warning: Use this method carefully, there is no prompting and the effect is
582    * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
583    * {@link #deleteTable(TableName)}
584    *
585    * @param pattern The pattern to match table names against
586    * @return Table descriptors for tables that couldn't be deleted
587    * @throws IOException
588    */
589   @Override
590   public HTableDescriptor[] deleteTables(Pattern pattern) throws IOException {
591     List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
592     for (HTableDescriptor table : listTables(pattern)) {
593       try {
594         deleteTable(table.getTableName());
595       } catch (IOException ex) {
596         LOG.info("Failed to delete table " + table.getTableName(), ex);
597         failed.add(table);
598       }
599     }
600     return failed.toArray(new HTableDescriptor[failed.size()]);
601   }
602 
603   @Override
604   public void truncateTable(final TableName tableName, final boolean preserveSplits)
605       throws IOException {
606     get(truncateTableAsync(tableName, preserveSplits), syncWaitTimeout, TimeUnit.MILLISECONDS);
607   }
608 
609   @Override
610   public Future<Void> truncateTableAsync(final TableName tableName, final boolean preserveSplits)
611       throws IOException {
612     TruncateTableResponse response =
613         executeCallable(new MasterCallable<TruncateTableResponse>(getConnection()) {
614           @Override
615           public TruncateTableResponse call(int callTimeout) throws ServiceException {
616             LOG.info("Started truncating " + tableName);
617             TruncateTableRequest req = RequestConverter.buildTruncateTableRequest(
618               tableName, preserveSplits, ng.getNonceGroup(), ng.newNonce());
619             return master.truncateTable(null, req);
620           }
621         });
622     return new TruncateTableFuture(this, tableName, preserveSplits, response);
623   }
624 
625   private static class TruncateTableFuture extends TableFuture<Void> {
626     private final boolean preserveSplits;
627 
628     public TruncateTableFuture(final HBaseAdmin admin, final TableName tableName,
629         final boolean preserveSplits, final TruncateTableResponse response) {
630       super(admin, tableName,
631              (response != null && response.hasProcId()) ? response.getProcId() : null);
632       this.preserveSplits = preserveSplits;
633     }
634 
635     @Override
636     public String getOperationType() {
637       return "TRUNCATE";
638     }
639 
640     @Override
641     protected Void waitOperationResult(final long deadlineTs) throws IOException, TimeoutException {
642       waitForTableEnabled(deadlineTs);
643       // once the table is enabled, we know the operation is done. so we can fetch the splitKeys
644       byte[][] splitKeys = preserveSplits ? getAdmin().getTableSplits(getTableName()) : null;
645       waitForAllRegionsOnline(deadlineTs, splitKeys);
646       return null;
647     }
648   }
649 
650   private byte[][] getTableSplits(final TableName tableName) throws IOException {
651     byte[][] splits = null;
652     try (RegionLocator locator = getConnection().getRegionLocator(tableName)) {
653       byte[][] startKeys = locator.getStartKeys();
654       if (startKeys.length == 1) {
655         return splits;
656       }
657       splits = new byte[startKeys.length - 1][];
658       for (int i = 1; i < startKeys.length; i++) {
659         splits[i - 1] = startKeys[i];
660       }
661     }
662     return splits;
663   }
664 
665   @Override
666   public void enableTable(final TableName tableName)
667   throws IOException {
668     get(enableTableAsync(tableName), syncWaitTimeout, TimeUnit.MILLISECONDS);
669   }
670 
671   /**
672    * Wait for the table to be enabled and available
673    * If enabling the table exceeds the retry period, an exception is thrown.
674    * @param tableName name of the table
675    * @throws IOException if a remote or network exception occurs or
676    *    table is not enabled after the retries period.
677    */
678   private void waitUntilTableIsEnabled(final TableName tableName) throws IOException {
679     boolean enabled = false;
680     long start = EnvironmentEdgeManager.currentTime();
681     for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
682       try {
683         enabled = isTableEnabled(tableName);
684       } catch (TableNotFoundException tnfe) {
685         // wait for table to be created
686         enabled = false;
687       }
688       enabled = enabled && isTableAvailable(tableName);
689       if (enabled) {
690         break;
691       }
692       long sleep = getPauseTime(tries);
693       if (LOG.isDebugEnabled()) {
694         LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " +
695           "enabled in " + tableName);
696       }
697       try {
698         Thread.sleep(sleep);
699       } catch (InterruptedException e) {
700         // Do this conversion rather than let it out because do not want to
701         // change the method signature.
702         throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
703       }
704     }
705     if (!enabled) {
706       long msec = EnvironmentEdgeManager.currentTime() - start;
707       throw new IOException("Table '" + tableName +
708         "' not yet enabled, after " + msec + "ms.");
709     }
710   }
711 
712   @Override
713   public Future<Void> enableTableAsync(final TableName tableName) throws IOException {
714     TableName.isLegalFullyQualifiedTableName(tableName.getName());
715     EnableTableResponse response = executeCallable(
716         new MasterCallable<EnableTableResponse>(getConnection()) {
717       @Override
718       public EnableTableResponse call(int callTimeout) throws ServiceException {
719         LOG.info("Started enable of " + tableName);
720         EnableTableRequest req =
721             RequestConverter.buildEnableTableRequest(tableName, ng.getNonceGroup(), ng.newNonce());
722         return master.enableTable(null,req);
723       }
724     });
725     return new EnableTableFuture(this, tableName, response);
726   }
727 
728   private static class EnableTableFuture extends TableFuture<Void> {
729     public EnableTableFuture(final HBaseAdmin admin, final TableName tableName,
730         final EnableTableResponse response) {
731       super(admin, tableName,
732               (response != null && response.hasProcId()) ? response.getProcId() : null);
733     }
734 
735     @Override
736     public String getOperationType() {
737       return "ENABLE";
738     }
739 
740     @Override
741     protected Void waitOperationResult(final long deadlineTs) throws IOException, TimeoutException {
742       waitForTableEnabled(deadlineTs);
743       return null;
744     }
745   }
746 
747   @Override
748   public HTableDescriptor[] enableTables(String regex) throws IOException {
749     return enableTables(Pattern.compile(regex));
750   }
751 
752   @Override
753   public HTableDescriptor[] enableTables(Pattern pattern) throws IOException {
754     List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
755     for (HTableDescriptor table : listTables(pattern)) {
756       if (isTableDisabled(table.getTableName())) {
757         try {
758           enableTable(table.getTableName());
759         } catch (IOException ex) {
760           LOG.info("Failed to enable table " + table.getTableName(), ex);
761           failed.add(table);
762         }
763       }
764     }
765     return failed.toArray(new HTableDescriptor[failed.size()]);
766   }
767 
768   @Override
769   public void disableTable(final TableName tableName)
770   throws IOException {
771     get(disableTableAsync(tableName), syncWaitTimeout, TimeUnit.MILLISECONDS);
772   }
773 
774   @Override
775   public Future<Void> disableTableAsync(final TableName tableName) throws IOException {
776     TableName.isLegalFullyQualifiedTableName(tableName.getName());
777     DisableTableResponse response = executeCallable(
778         new MasterCallable<DisableTableResponse>(getConnection()) {
779       @Override
780       public DisableTableResponse call(int callTimeout) throws ServiceException {
781         LOG.info("Started disable of " + tableName);
782         DisableTableRequest req =
783             RequestConverter.buildDisableTableRequest(tableName, ng.getNonceGroup(), ng.newNonce());
784         return master.disableTable(null, req);
785       }
786     });
787     return new DisableTableFuture(this, tableName, response);
788   }
789 
790   private static class DisableTableFuture extends TableFuture<Void> {
791     public DisableTableFuture(final HBaseAdmin admin, final TableName tableName,
792         final DisableTableResponse response) {
793       super(admin, tableName,
794               (response != null && response.hasProcId()) ? response.getProcId() : null);
795     }
796 
797     @Override
798     public String getOperationType() {
799       return "DISABLE";
800     }
801 
802     @Override
803     protected Void waitOperationResult(long deadlineTs) throws IOException, TimeoutException {
804       waitForTableDisabled(deadlineTs);
805       return null;
806     }
807   }
808 
809   @Override
810   public HTableDescriptor[] disableTables(String regex) throws IOException {
811     return disableTables(Pattern.compile(regex));
812   }
813 
814   @Override
815   public HTableDescriptor[] disableTables(Pattern pattern) throws IOException {
816     List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
817     for (HTableDescriptor table : listTables(pattern)) {
818       if (isTableEnabled(table.getTableName())) {
819         try {
820           disableTable(table.getTableName());
821         } catch (IOException ex) {
822           LOG.info("Failed to disable table " + table.getTableName(), ex);
823           failed.add(table);
824         }
825       }
826     }
827     return failed.toArray(new HTableDescriptor[failed.size()]);
828   }
829 
830   @Override
831   public boolean isTableEnabled(final TableName tableName) throws IOException {
832     checkTableExists(tableName);
833     return executeCallable(new ConnectionCallable<Boolean>(getConnection()) {
834       @Override
835       public Boolean call(int callTimeout) throws ServiceException, IOException {
836         TableState tableState = MetaTableAccessor.getTableState(connection, tableName);
837         if (tableState == null)
838           throw new TableNotFoundException(tableName);
839         return tableState.inStates(TableState.State.ENABLED);
840       }
841     });
842   }
843 
844   @Override
845   public boolean isTableDisabled(TableName tableName) throws IOException {
846     checkTableExists(tableName);
847     return connection.isTableDisabled(tableName);
848   }
849 
850   @Override
851   public boolean isTableAvailable(TableName tableName) throws IOException {
852     return connection.isTableAvailable(tableName);
853   }
854 
855   @Override
856   public boolean isTableAvailable(TableName tableName, byte[][] splitKeys) throws IOException {
857     return connection.isTableAvailable(tableName, splitKeys);
858   }
859 
860   @Override
861   public Pair<Integer, Integer> getAlterStatus(final TableName tableName) throws IOException {
862     return executeCallable(new MasterCallable<Pair<Integer, Integer>>(getConnection()) {
863       @Override
864       public Pair<Integer, Integer> call(int callTimeout) throws ServiceException {
865         GetSchemaAlterStatusRequest req = RequestConverter
866             .buildGetSchemaAlterStatusRequest(tableName);
867         GetSchemaAlterStatusResponse ret = master.getSchemaAlterStatus(null, req);
868         Pair<Integer, Integer> pair = new Pair<>(ret.getYetToUpdateRegions(),
869             ret.getTotalRegions());
870         return pair;
871       }
872     });
873   }
874 
875   @Override
876   public Pair<Integer, Integer> getAlterStatus(final byte[] tableName) throws IOException {
877     return getAlterStatus(TableName.valueOf(tableName));
878   }
879 
880   /**
881    * {@inheritDoc}
882    * @deprecated Since 2.0. Will be removed in 3.0. Use
883    *     {@link #addColumnFamily(TableName, HColumnDescriptor)} instead.
884    */
885   @Override
886   @Deprecated
887   public void addColumn(final TableName tableName, final HColumnDescriptor columnFamily)
888   throws IOException {
889     addColumnFamily(tableName, columnFamily);
890   }
891 
892   @Override
893   public Future<Void> addColumnFamily(final TableName tableName,
894       final HColumnDescriptor columnFamily) throws IOException {
895     AddColumnResponse response =
896         executeCallable(new MasterCallable<AddColumnResponse>(getConnection()) {
897           @Override
898           public AddColumnResponse call(int callTimeout) throws ServiceException {
899             AddColumnRequest req =
900                 RequestConverter.buildAddColumnRequest(tableName, columnFamily, ng.getNonceGroup(),
901                   ng.newNonce());
902             return master.addColumn(null, req);
903           }
904         });
905     return new AddColumnFamilyFuture(this, tableName, response);
906   }
907 
908   private static class AddColumnFamilyFuture extends ModifyTableFuture {
909     public AddColumnFamilyFuture(final HBaseAdmin admin, final TableName tableName,
910         final AddColumnResponse response) {
911       super(admin, tableName, (response != null && response.hasProcId()) ? response.getProcId()
912           : null);
913     }
914 
915     @Override
916     public String getOperationType() {
917       return "ADD_COLUMN_FAMILY";
918     }
919   }
920 
921   /**
922    * {@inheritDoc}
923    * @deprecated Since 2.0. Will be removed in 3.0. Use
924    *     {@link #deleteColumnFamily(TableName, byte[])} instead.
925    */
926   @Override
927   @Deprecated
928   public void deleteColumn(final TableName tableName, final byte[] columnFamily)
929   throws IOException {
930     deleteColumnFamily(tableName, columnFamily);
931   }
932 
933   @Override
934   public Future<Void> deleteColumnFamily(final TableName tableName, final byte[] columnFamily)
935       throws IOException {
936     DeleteColumnResponse response =
937         executeCallable(new MasterCallable<DeleteColumnResponse>(getConnection()) {
938           @Override
939           public DeleteColumnResponse call(int callTimeout) throws ServiceException {
940             DeleteColumnRequest req =
941                 RequestConverter.buildDeleteColumnRequest(tableName, columnFamily,
942                   ng.getNonceGroup(), ng.newNonce());
943             master.deleteColumn(null, req);
944             return null;
945           }
946         });
947     return new DeleteColumnFamilyFuture(this, tableName, response);
948   }
949 
950   private static class DeleteColumnFamilyFuture extends ModifyTableFuture {
951     public DeleteColumnFamilyFuture(final HBaseAdmin admin, final TableName tableName,
952         final DeleteColumnResponse response) {
953       super(admin, tableName, (response != null && response.hasProcId()) ? response.getProcId()
954           : null);
955     }
956 
957     @Override
958     public String getOperationType() {
959       return "DELETE_COLUMN_FAMILY";
960     }
961   }
962 
963   /**
964    * {@inheritDoc}
965    * @deprecated As of 2.0. Will be removed in 3.0. Use
966    *     {@link #modifyColumnFamily(TableName, HColumnDescriptor)} instead.
967    */
968   @Override
969   @Deprecated
970   public void modifyColumn(final TableName tableName, final HColumnDescriptor columnFamily)
971   throws IOException {
972     modifyColumnFamily(tableName, columnFamily);
973   }
974 
975   @Override
976   public Future<Void> modifyColumnFamily(final TableName tableName,
977       final HColumnDescriptor columnFamily) throws IOException {
978     ModifyColumnResponse response =
979         executeCallable(new MasterCallable<ModifyColumnResponse>(getConnection()) {
980           @Override
981           public ModifyColumnResponse call(int callTimeout) throws ServiceException {
982             ModifyColumnRequest req =
983                 RequestConverter.buildModifyColumnRequest(tableName, columnFamily,
984                   ng.getNonceGroup(), ng.newNonce());
985             master.modifyColumn(null, req);
986             return null;
987           }
988         });
989     return new ModifyColumnFamilyFuture(this, tableName, response);
990   }
991 
992   private static class ModifyColumnFamilyFuture extends ModifyTableFuture {
993     public ModifyColumnFamilyFuture(final HBaseAdmin admin, final TableName tableName,
994         final ModifyColumnResponse response) {
995       super(admin, tableName, (response != null && response.hasProcId()) ? response.getProcId()
996           : null);
997     }
998 
999     @Override
1000     public String getOperationType() {
1001       return "MODIFY_COLUMN_FAMILY";
1002     }
1003   }
1004 
1005   @Override
1006   public void closeRegion(final String regionname, final String serverName) throws IOException {
1007     closeRegion(Bytes.toBytes(regionname), serverName);
1008   }
1009 
1010   @Override
1011   public void closeRegion(final byte [] regionname, final String serverName) throws IOException {
1012     if (serverName != null) {
1013       Pair<HRegionInfo, ServerName> pair = MetaTableAccessor.getRegion(connection, regionname);
1014       if (pair == null || pair.getFirst() == null) {
1015         throw new UnknownRegionException(Bytes.toStringBinary(regionname));
1016       } else {
1017         closeRegion(ServerName.valueOf(serverName), pair.getFirst());
1018       }
1019     } else {
1020       Pair<HRegionInfo, ServerName> pair = MetaTableAccessor.getRegion(connection, regionname);
1021       if (pair == null) {
1022         throw new UnknownRegionException(Bytes.toStringBinary(regionname));
1023       } else if (pair.getSecond() == null) {
1024         throw new NoServerForRegionException(Bytes.toStringBinary(regionname));
1025       } else {
1026         closeRegion(pair.getSecond(), pair.getFirst());
1027       }
1028     }
1029   }
1030 
1031   @Override
1032   public boolean closeRegionWithEncodedRegionName(final String encodedRegionName,
1033       final String serverName) throws IOException {
1034     if (null == serverName || ("").equals(serverName.trim())) {
1035       throw new IllegalArgumentException(
1036           "The servername cannot be null or empty.");
1037     }
1038     ServerName sn = ServerName.valueOf(serverName);
1039     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1040     // Close the region without updating zk state.
1041     CloseRegionRequest request =
1042       RequestConverter.buildCloseRegionRequest(sn, encodedRegionName);
1043     try {
1044       CloseRegionResponse response = admin.closeRegion(null, request);
1045       boolean isRegionClosed = response.getClosed();
1046       if (false == isRegionClosed) {
1047         LOG.error("Not able to close the region " + encodedRegionName + ".");
1048       }
1049       return isRegionClosed;
1050     } catch (ServiceException se) {
1051       throw ProtobufUtil.getRemoteException(se);
1052     }
1053   }
1054 
1055   @Override
1056   public void closeRegion(final ServerName sn, final HRegionInfo hri) throws IOException {
1057     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1058     // Close the region without updating zk state.
1059     ProtobufUtil.closeRegion(admin, sn, hri.getRegionName());
1060   }
1061 
1062   @Override
1063   public List<HRegionInfo> getOnlineRegions(final ServerName sn) throws IOException {
1064     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1065     return ProtobufUtil.getOnlineRegions(admin);
1066   }
1067 
1068   @Override
1069   public void flush(final TableName tableName) throws IOException {
1070     checkTableExists(tableName);
1071     if (isTableDisabled(tableName)) {
1072       LOG.info("Table is disabled: " + tableName.getNameAsString());
1073       return;
1074     }
1075     execProcedure("flush-table-proc", tableName.getNameAsString(),
1076       new HashMap<String, String>());
1077   }
1078 
1079   @Override
1080   public void flushRegion(final byte[] regionName) throws IOException {
1081     Pair<HRegionInfo, ServerName> regionServerPair = getRegion(regionName);
1082     if (regionServerPair == null) {
1083       throw new IllegalArgumentException("Unknown regionname: " + Bytes.toStringBinary(regionName));
1084     }
1085     if (regionServerPair.getSecond() == null) {
1086       throw new NoServerForRegionException(Bytes.toStringBinary(regionName));
1087     }
1088     HRegionInfo hRegionInfo = regionServerPair.getFirst();
1089     ServerName serverName = regionServerPair.getSecond();
1090     AdminService.BlockingInterface admin = this.connection.getAdmin(serverName);
1091     FlushRegionRequest request =
1092         RequestConverter.buildFlushRegionRequest(hRegionInfo.getRegionName());
1093     try {
1094       admin.flushRegion(null, request);
1095     } catch (ServiceException se) {
1096       throw ProtobufUtil.getRemoteException(se);
1097     }
1098   }
1099 
1100   private void flush(final ServerName sn, final HRegionInfo hri)
1101   throws IOException {
1102     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1103     FlushRegionRequest request =
1104       RequestConverter.buildFlushRegionRequest(hri.getRegionName());
1105     try {
1106       admin.flushRegion(null, request);
1107     } catch (ServiceException se) {
1108       throw ProtobufUtil.getRemoteException(se);
1109     }
1110   }
1111 
1112   /**
1113    * {@inheritDoc}
1114    */
1115   @Override
1116   public void compact(final TableName tableName)
1117     throws IOException {
1118     compact(tableName, null, false, CompactType.NORMAL);
1119   }
1120 
1121   @Override
1122   public void compactRegion(final byte[] regionName)
1123     throws IOException {
1124     compactRegion(regionName, null, false);
1125   }
1126 
1127   /**
1128    * {@inheritDoc}
1129    */
1130   @Override
1131   public void compact(final TableName tableName, final byte[] columnFamily)
1132     throws IOException {
1133     compact(tableName, columnFamily, false, CompactType.NORMAL);
1134   }
1135 
1136   /**
1137    * {@inheritDoc}
1138    */
1139   @Override
1140   public void compactRegion(final byte[] regionName, final byte[] columnFamily)
1141     throws IOException {
1142     compactRegion(regionName, columnFamily, false);
1143   }
1144 
1145   /**
1146    * {@inheritDoc}
1147    */
1148   @Override
1149   public void compactRegionServer(final ServerName sn, boolean major)
1150   throws IOException, InterruptedException {
1151     for (HRegionInfo region : getOnlineRegions(sn)) {
1152       compact(sn, region, major, null);
1153     }
1154   }
1155 
1156   @Override
1157   public void majorCompact(final TableName tableName)
1158   throws IOException {
1159     compact(tableName, null, true, CompactType.NORMAL);
1160   }
1161 
1162   @Override
1163   public void majorCompactRegion(final byte[] regionName)
1164   throws IOException {
1165     compactRegion(regionName, null, true);
1166   }
1167 
1168   /**
1169    * {@inheritDoc}
1170    */
1171   @Override
1172   public void majorCompact(final TableName tableName, final byte[] columnFamily)
1173   throws IOException {
1174     compact(tableName, columnFamily, true, CompactType.NORMAL);
1175   }
1176 
1177   @Override
1178   public void majorCompactRegion(final byte[] regionName, final byte[] columnFamily)
1179   throws IOException {
1180     compactRegion(regionName, columnFamily, true);
1181   }
1182 
1183   /**
1184    * Compact a table.
1185    * Asynchronous operation.
1186    *
1187    * @param tableName table or region to compact
1188    * @param columnFamily column family within a table or region
1189    * @param major True if we are to do a major compaction.
1190    * @throws IOException if a remote or network exception occurs
1191    * @throws InterruptedException
1192    */
1193   private void compact(final TableName tableName, final byte[] columnFamily,final boolean major,
1194                        CompactType compactType) throws IOException {
1195     switch (compactType) {
1196       case MOB:
1197         ServerName master = getMasterAddress();
1198         compact(master, getMobRegionInfo(tableName), major, columnFamily);
1199         break;
1200       case NORMAL:
1201       default:
1202         ZooKeeperWatcher zookeeper = null;
1203         try {
1204           checkTableExists(tableName);
1205           zookeeper = new ZooKeeperWatcher(conf, ZK_IDENTIFIER_PREFIX + connection.toString(),
1206                   new ThrowableAbortable());
1207           List<Pair<HRegionInfo, ServerName>> pairs;
1208           if (TableName.META_TABLE_NAME.equals(tableName)) {
1209             pairs = new MetaTableLocator().getMetaRegionsAndLocations(zookeeper);
1210           } else {
1211             pairs = MetaTableAccessor.getTableRegionsAndLocations(connection, tableName);
1212           }
1213           for (Pair<HRegionInfo, ServerName> pair: pairs) {
1214             if (pair.getFirst().isOffline()) continue;
1215             if (pair.getSecond() == null) continue;
1216             try {
1217               compact(pair.getSecond(), pair.getFirst(), major, columnFamily);
1218             } catch (NotServingRegionException e) {
1219               if (LOG.isDebugEnabled()) {
1220                 LOG.debug("Trying to" + (major ? " major" : "") + " compact " +
1221                         pair.getFirst() + ": " +
1222                         StringUtils.stringifyException(e));
1223               }
1224             }
1225           }
1226         } finally {
1227           if (zookeeper != null) {
1228             zookeeper.close();
1229           }
1230         }
1231         break;
1232     }
1233   }
1234 
1235   /**
1236    * Compact an individual region.
1237    * Asynchronous operation.
1238    *
1239    * @param regionName region to compact
1240    * @param columnFamily column family within a table or region
1241    * @param major True if we are to do a major compaction.
1242    * @throws IOException if a remote or network exception occurs
1243    * @throws InterruptedException
1244    */
1245   private void compactRegion(final byte[] regionName, final byte[] columnFamily,final boolean major)
1246   throws IOException {
1247     Pair<HRegionInfo, ServerName> regionServerPair = getRegion(regionName);
1248     if (regionServerPair == null) {
1249       throw new IllegalArgumentException("Invalid region: " + Bytes.toStringBinary(regionName));
1250     }
1251     if (regionServerPair.getSecond() == null) {
1252       throw new NoServerForRegionException(Bytes.toStringBinary(regionName));
1253     }
1254     compact(regionServerPair.getSecond(), regionServerPair.getFirst(), major, columnFamily);
1255   }
1256 
1257   private void compact(final ServerName sn, final HRegionInfo hri,
1258       final boolean major, final byte [] family)
1259   throws IOException {
1260     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1261     CompactRegionRequest request =
1262       RequestConverter.buildCompactRegionRequest(hri.getRegionName(), major, family);
1263     try {
1264       admin.compactRegion(null, request);
1265     } catch (ServiceException se) {
1266       throw ProtobufUtil.getRemoteException(se);
1267     }
1268   }
1269 
1270   @Override
1271   public void move(final byte [] encodedRegionName, final byte [] destServerName)
1272       throws IOException {
1273 
1274     executeCallable(new MasterCallable<Void>(getConnection()) {
1275       @Override
1276       public Void call(int callTimeout) throws ServiceException {
1277         try {
1278           MoveRegionRequest request =
1279               RequestConverter.buildMoveRegionRequest(encodedRegionName, destServerName);
1280             master.moveRegion(null, request);
1281         } catch (DeserializationException de) {
1282           LOG.error("Could not parse destination server name: " + de);
1283           throw new ServiceException(new DoNotRetryIOException(de));
1284         }
1285         return null;
1286       }
1287     });
1288   }
1289 
1290   @Override
1291   public void assign(final byte[] regionName) throws MasterNotRunningException,
1292       ZooKeeperConnectionException, IOException {
1293     final byte[] toBeAssigned = getRegionName(regionName);
1294     executeCallable(new MasterCallable<Void>(getConnection()) {
1295       @Override
1296       public Void call(int callTimeout) throws ServiceException {
1297         AssignRegionRequest request =
1298           RequestConverter.buildAssignRegionRequest(toBeAssigned);
1299         master.assignRegion(null,request);
1300         return null;
1301       }
1302     });
1303   }
1304 
1305   @Override
1306   public void unassign(final byte [] regionName, final boolean force)
1307   throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
1308     final byte[] toBeUnassigned = getRegionName(regionName);
1309     executeCallable(new MasterCallable<Void>(getConnection()) {
1310       @Override
1311       public Void call(int callTimeout) throws ServiceException {
1312         UnassignRegionRequest request =
1313           RequestConverter.buildUnassignRegionRequest(toBeUnassigned, force);
1314         master.unassignRegion(null, request);
1315         return null;
1316       }
1317     });
1318   }
1319 
1320   @Override
1321   public void offline(final byte [] regionName)
1322   throws IOException {
1323     executeCallable(new MasterCallable<Void>(getConnection()) {
1324       @Override
1325       public Void call(int callTimeout) throws ServiceException {
1326         master.offlineRegion(null,RequestConverter.buildOfflineRegionRequest(regionName));
1327         return null;
1328       }
1329     });
1330   }
1331 
1332   @Override
1333   public boolean setBalancerRunning(final boolean on, final boolean synchronous)
1334   throws IOException {
1335     return executeCallable(new MasterCallable<Boolean>(getConnection()) {
1336       @Override
1337       public Boolean call(int callTimeout) throws ServiceException {
1338         SetBalancerRunningRequest req =
1339             RequestConverter.buildSetBalancerRunningRequest(on, synchronous);
1340         return master.setBalancerRunning(null, req).getPrevBalanceValue();
1341       }
1342     });
1343   }
1344 
1345   @Override
1346   public boolean balancer() throws IOException {
1347     return executeCallable(new MasterCallable<Boolean>(getConnection()) {
1348       @Override
1349       public Boolean call(int callTimeout) throws ServiceException {
1350         return master.balance(null, RequestConverter.buildBalanceRequest(false)).getBalancerRan();
1351       }
1352     });
1353   }
1354 
1355   @Override
1356   public boolean balancer(final boolean force) throws IOException {
1357     return executeCallable(new MasterCallable<Boolean>(getConnection()) {
1358       @Override
1359       public Boolean call(int callTimeout) throws ServiceException {
1360         return master.balance(null, RequestConverter.buildBalanceRequest(force)).getBalancerRan();
1361       }
1362     });
1363   }
1364 
1365   @Override
1366   public boolean isBalancerEnabled() throws IOException {
1367     return executeCallable(new MasterCallable<Boolean>(getConnection()) {
1368       @Override
1369       public Boolean call(int callTimeout) throws ServiceException {
1370         return master.isBalancerEnabled(null, RequestConverter.buildIsBalancerEnabledRequest())
1371             .getEnabled();
1372       }
1373     });
1374   }
1375 
1376   @Override
1377   public boolean normalize() throws IOException {
1378     return executeCallable(new MasterCallable<Boolean>(getConnection()) {
1379       @Override
1380       public Boolean call(int callTimeout) throws ServiceException {
1381         return master.normalize(null,
1382           RequestConverter.buildNormalizeRequest()).getNormalizerRan();
1383       }
1384     });
1385   }
1386 
1387   @Override
1388   public boolean isNormalizerEnabled() throws IOException {
1389     return executeCallable(new MasterCallable<Boolean>(getConnection()) {
1390       @Override
1391       public Boolean call(int callTimeout) throws ServiceException {
1392         return master.isNormalizerEnabled(null,
1393           RequestConverter.buildIsNormalizerEnabledRequest()).getEnabled();
1394       }
1395     });
1396   }
1397 
1398   @Override
1399   public boolean setNormalizerRunning(final boolean on) throws IOException {
1400     return executeCallable(new MasterCallable<Boolean>(getConnection()) {
1401       @Override
1402       public Boolean call(int callTimeout) throws ServiceException {
1403         SetNormalizerRunningRequest req =
1404           RequestConverter.buildSetNormalizerRunningRequest(on);
1405         return master.setNormalizerRunning(null, req).getPrevNormalizerValue();
1406       }
1407     });
1408   }
1409 
1410   @Override
1411   public boolean enableCatalogJanitor(final boolean enable) throws IOException {
1412     return executeCallable(new MasterCallable<Boolean>(getConnection()) {
1413       @Override
1414       public Boolean call(int callTimeout) throws ServiceException {
1415         return master.enableCatalogJanitor(null,
1416           RequestConverter.buildEnableCatalogJanitorRequest(enable)).getPrevValue();
1417       }
1418     });
1419   }
1420 
1421   @Override
1422   public int runCatalogScan() throws IOException {
1423     return executeCallable(new MasterCallable<Integer>(getConnection()) {
1424       @Override
1425       public Integer call(int callTimeout) throws ServiceException {
1426         return master.runCatalogScan(null,
1427           RequestConverter.buildCatalogScanRequest()).getScanResult();
1428       }
1429     });
1430   }
1431 
1432   @Override
1433   public boolean isCatalogJanitorEnabled() throws IOException {
1434     return executeCallable(new MasterCallable<Boolean>(getConnection()) {
1435       @Override
1436       public Boolean call(int callTimeout) throws ServiceException {
1437         return master.isCatalogJanitorEnabled(null,
1438           RequestConverter.buildIsCatalogJanitorEnabledRequest()).getValue();
1439       }
1440     });
1441   }
1442 
1443   private boolean isEncodedRegionName(byte[] regionName) throws IOException {
1444     try {
1445       HRegionInfo.parseRegionName(regionName);
1446       return false;
1447     } catch (IOException e) {
1448       if (StringUtils.stringifyException(e)
1449         .contains(HRegionInfo.INVALID_REGION_NAME_FORMAT_MESSAGE)) {
1450         return true;
1451       }
1452       throw e;
1453     }
1454   }
1455 
1456   /**
1457    * Merge two regions. Asynchronous operation.
1458    * @param nameOfRegionA encoded or full name of region a
1459    * @param nameOfRegionB encoded or full name of region b
1460    * @param forcible true if do a compulsory merge, otherwise we will only merge
1461    *          two adjacent regions
1462    * @throws IOException
1463    */
1464   @Override
1465   public void mergeRegions(final byte[] nameOfRegionA,
1466       final byte[] nameOfRegionB, final boolean forcible)
1467       throws IOException {
1468     final byte[] encodedNameOfRegionA = isEncodedRegionName(nameOfRegionA) ?
1469       nameOfRegionA : HRegionInfo.encodeRegionName(nameOfRegionA).getBytes();
1470     final byte[] encodedNameOfRegionB = isEncodedRegionName(nameOfRegionB) ?
1471       nameOfRegionB : HRegionInfo.encodeRegionName(nameOfRegionB).getBytes();
1472 
1473     Pair<HRegionInfo, ServerName> pair = getRegion(nameOfRegionA);
1474     if (pair != null && pair.getFirst().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID)
1475       throw new IllegalArgumentException("Can't invoke merge on non-default regions directly");
1476     pair = getRegion(nameOfRegionB);
1477     if (pair != null && pair.getFirst().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID)
1478       throw new IllegalArgumentException("Can't invoke merge on non-default regions directly");
1479     executeCallable(new MasterCallable<Void>(getConnection()) {
1480       @Override
1481       public Void call(int callTimeout) throws ServiceException {
1482         try {
1483           DispatchMergingRegionsRequest request = RequestConverter
1484               .buildDispatchMergingRegionsRequest(encodedNameOfRegionA,
1485                 encodedNameOfRegionB, forcible);
1486           master.dispatchMergingRegions(null, request);
1487         } catch (DeserializationException de) {
1488           LOG.error("Could not parse destination server name: " + de);
1489         }
1490         return null;
1491       }
1492     });
1493   }
1494 
1495   @Override
1496   public void split(final TableName tableName) throws IOException {
1497     split(tableName, null);
1498   }
1499 
1500   @Override
1501   public void splitRegion(final byte[] regionName) throws IOException {
1502     splitRegion(regionName, null);
1503   }
1504 
1505   /**
1506    * {@inheritDoc}
1507    */
1508   @Override
1509   public void split(final TableName tableName, final byte [] splitPoint) throws IOException {
1510     ZooKeeperWatcher zookeeper = null;
1511     try {
1512       checkTableExists(tableName);
1513       zookeeper = new ZooKeeperWatcher(conf, ZK_IDENTIFIER_PREFIX + connection.toString(),
1514         new ThrowableAbortable());
1515       List<Pair<HRegionInfo, ServerName>> pairs;
1516       if (TableName.META_TABLE_NAME.equals(tableName)) {
1517         pairs = new MetaTableLocator().getMetaRegionsAndLocations(zookeeper);
1518       } else {
1519         pairs = MetaTableAccessor.getTableRegionsAndLocations(connection, tableName);
1520       }
1521       for (Pair<HRegionInfo, ServerName> pair: pairs) {
1522         // May not be a server for a particular row
1523         if (pair.getSecond() == null) continue;
1524         HRegionInfo r = pair.getFirst();
1525         // check for parents
1526         if (r.isSplitParent()) continue;
1527         // if a split point given, only split that particular region
1528         if (r.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID ||
1529            (splitPoint != null && !r.containsRow(splitPoint))) continue;
1530         // call out to region server to do split now
1531         split(pair.getSecond(), pair.getFirst(), splitPoint);
1532       }
1533     } finally {
1534       if (zookeeper != null) {
1535         zookeeper.close();
1536       }
1537     }
1538   }
1539 
1540   @Override
1541   public void splitRegion(final byte[] regionName, final byte [] splitPoint) throws IOException {
1542     Pair<HRegionInfo, ServerName> regionServerPair = getRegion(regionName);
1543     if (regionServerPair == null) {
1544       throw new IllegalArgumentException("Invalid region: " + Bytes.toStringBinary(regionName));
1545     }
1546     if (regionServerPair.getFirst() != null &&
1547         regionServerPair.getFirst().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1548       throw new IllegalArgumentException("Can't split replicas directly. "
1549           + "Replicas are auto-split when their primary is split.");
1550     }
1551     if (regionServerPair.getSecond() == null) {
1552       throw new NoServerForRegionException(Bytes.toStringBinary(regionName));
1553     }
1554     split(regionServerPair.getSecond(), regionServerPair.getFirst(), splitPoint);
1555   }
1556 
1557   @VisibleForTesting
1558   public void split(final ServerName sn, final HRegionInfo hri,
1559       byte[] splitPoint) throws IOException {
1560     if (hri.getStartKey() != null && splitPoint != null &&
1561          Bytes.compareTo(hri.getStartKey(), splitPoint) == 0) {
1562        throw new IOException("should not give a splitkey which equals to startkey!");
1563     }
1564     // TODO: This is not executed via retries
1565     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1566     ProtobufUtil.split(admin, hri, splitPoint);
1567   }
1568 
1569   @Override
1570   public Future<Void> modifyTable(final TableName tableName, final HTableDescriptor htd)
1571   throws IOException {
1572     if (!tableName.equals(htd.getTableName())) {
1573       throw new IllegalArgumentException("the specified table name '" + tableName +
1574         "' doesn't match with the HTD one: " + htd.getTableName());
1575     }
1576 
1577     ModifyTableResponse response = executeCallable(
1578         new MasterCallable<ModifyTableResponse>(getConnection()) {
1579       @Override
1580       public ModifyTableResponse call(int callTimeout) throws ServiceException {
1581         ModifyTableRequest request = RequestConverter.buildModifyTableRequest(
1582           tableName, htd, ng.getNonceGroup(), ng.newNonce());
1583         return master.modifyTable(null, request);
1584       }
1585     });
1586 
1587     return new ModifyTableFuture(this, tableName, response);
1588   }
1589 
1590   private static class ModifyTableFuture extends TableFuture<Void> {
1591     public ModifyTableFuture(final HBaseAdmin admin, final TableName tableName,
1592         final ModifyTableResponse response) {
1593       super(admin, tableName,
1594           (response != null && response.hasProcId()) ? response.getProcId() : null);
1595     }
1596 
1597     public ModifyTableFuture(final HBaseAdmin admin, final TableName tableName, final Long procId) {
1598       super(admin, tableName, procId);
1599     }
1600 
1601     @Override
1602     public String getOperationType() {
1603       return "MODIFY";
1604     }
1605 
1606     @Override
1607     protected Void postOperationResult(final Void result, final long deadlineTs)
1608         throws IOException, TimeoutException {
1609       // The modify operation on the table is asynchronous on the server side irrespective
1610       // of whether Procedure V2 is supported or not. So, we wait in the client till
1611       // all regions get updated.
1612       waitForSchemaUpdate(deadlineTs);
1613       return result;
1614     }
1615   }
1616 
1617   /**
1618    * @param regionName Name of a region.
1619    * @return a pair of HRegionInfo and ServerName if <code>regionName</code> is
1620    *  a verified region name (we call {@link
1621    *  MetaTableAccessor#getRegionLocation(HConnection, byte[])}
1622    *  else null.
1623    * Throw IllegalArgumentException if <code>regionName</code> is null.
1624    * @throws IOException
1625    */
1626   Pair<HRegionInfo, ServerName> getRegion(final byte[] regionName) throws IOException {
1627     if (regionName == null) {
1628       throw new IllegalArgumentException("Pass a table name or region name");
1629     }
1630     Pair<HRegionInfo, ServerName> pair =
1631       MetaTableAccessor.getRegion(connection, regionName);
1632     if (pair == null) {
1633       final AtomicReference<Pair<HRegionInfo, ServerName>> result =
1634         new AtomicReference<Pair<HRegionInfo, ServerName>>(null);
1635       final String encodedName = Bytes.toString(regionName);
1636       MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {
1637         @Override
1638         public boolean visit(Result data) throws IOException {
1639           HRegionInfo info = MetaTableAccessor.getHRegionInfo(data);
1640           if (info == null) {
1641             LOG.warn("No serialized HRegionInfo in " + data);
1642             return true;
1643           }
1644           RegionLocations rl = MetaTableAccessor.getRegionLocations(data);
1645           boolean matched = false;
1646           ServerName sn = null;
1647           if (rl != null) {
1648             for (HRegionLocation h : rl.getRegionLocations()) {
1649               if (h != null && encodedName.equals(h.getRegionInfo().getEncodedName())) {
1650                 sn = h.getServerName();
1651                 info = h.getRegionInfo();
1652                 matched = true;
1653               }
1654             }
1655           }
1656           if (!matched) return true;
1657           result.set(new Pair<HRegionInfo, ServerName>(info, sn));
1658           return false; // found the region, stop
1659         }
1660       };
1661 
1662       MetaTableAccessor.fullScanRegions(connection, visitor);
1663       pair = result.get();
1664     }
1665     return pair;
1666   }
1667 
1668   /**
1669    * If the input is a region name, it is returned as is. If it's an
1670    * encoded region name, the corresponding region is found from meta
1671    * and its region name is returned. If we can't find any region in
1672    * meta matching the input as either region name or encoded region
1673    * name, the input is returned as is. We don't throw unknown
1674    * region exception.
1675    */
1676   private byte[] getRegionName(
1677       final byte[] regionNameOrEncodedRegionName) throws IOException {
1678     if (Bytes.equals(regionNameOrEncodedRegionName,
1679         HRegionInfo.FIRST_META_REGIONINFO.getRegionName())
1680           || Bytes.equals(regionNameOrEncodedRegionName,
1681             HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes())) {
1682       return HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
1683     }
1684     byte[] tmp = regionNameOrEncodedRegionName;
1685     Pair<HRegionInfo, ServerName> regionServerPair = getRegion(regionNameOrEncodedRegionName);
1686     if (regionServerPair != null && regionServerPair.getFirst() != null) {
1687       tmp = regionServerPair.getFirst().getRegionName();
1688     }
1689     return tmp;
1690   }
1691 
1692   /**
1693    * Check if table exists or not
1694    * @param tableName Name of a table.
1695    * @return tableName instance
1696    * @throws IOException if a remote or network exception occurs.
1697    * @throws TableNotFoundException if table does not exist.
1698    */
1699   private TableName checkTableExists(final TableName tableName)
1700       throws IOException {
1701     return executeCallable(new ConnectionCallable<TableName>(getConnection()) {
1702       @Override
1703       public TableName call(int callTimeout) throws ServiceException, IOException {
1704         if (!MetaTableAccessor.tableExists(connection, tableName)) {
1705           throw new TableNotFoundException(tableName);
1706         }
1707         return tableName;
1708       }
1709     });
1710   }
1711 
1712   @Override
1713   public synchronized void shutdown() throws IOException {
1714     executeCallable(new MasterCallable<Void>(getConnection()) {
1715       @Override
1716       public Void call(int callTimeout) throws ServiceException {
1717         master.shutdown(null,ShutdownRequest.newBuilder().build());
1718         return null;
1719       }
1720     });
1721   }
1722 
1723   @Override
1724   public synchronized void stopMaster() throws IOException {
1725     executeCallable(new MasterCallable<Void>(getConnection()) {
1726       @Override
1727       public Void call(int callTimeout) throws ServiceException {
1728         master.stopMaster(null, StopMasterRequest.newBuilder().build());
1729         return null;
1730       }
1731     });
1732   }
1733 
1734   @Override
1735   public synchronized void stopRegionServer(final String hostnamePort)
1736   throws IOException {
1737     String hostname = Addressing.parseHostname(hostnamePort);
1738     int port = Addressing.parsePort(hostnamePort);
1739     AdminService.BlockingInterface admin =
1740       this.connection.getAdmin(ServerName.valueOf(hostname, port, 0));
1741     StopServerRequest request = RequestConverter.buildStopServerRequest(
1742       "Called by admin client " + this.connection.toString());
1743     try {
1744       admin.stopServer(null, request);
1745     } catch (ServiceException se) {
1746       throw ProtobufUtil.getRemoteException(se);
1747     }
1748   }
1749 
1750   @Override
1751   public ClusterStatus getClusterStatus() throws IOException {
1752     return executeCallable(new MasterCallable<ClusterStatus>(getConnection()) {
1753       @Override
1754       public ClusterStatus call(int callTimeout) throws ServiceException {
1755         GetClusterStatusRequest req = RequestConverter.buildGetClusterStatusRequest();
1756         return ClusterStatus.convert(master.getClusterStatus(null, req).getClusterStatus());
1757       }
1758     });
1759   }
1760 
1761   @Override
1762   public Configuration getConfiguration() {
1763     return this.conf;
1764   }
1765 
1766   /**
1767    * Do a get with a timeout against the passed in <code>future<code>.
1768    */
1769   private static <T> T get(final Future<T> future, final long timeout, final TimeUnit units)
1770   throws IOException {
1771     try {
1772       // TODO: how long should we wait? Spin forever?
1773       return future.get(timeout, units);
1774     } catch (InterruptedException e) {
1775       throw new InterruptedIOException("Interrupt while waiting on " + future);
1776     } catch (TimeoutException e) {
1777       throw new TimeoutIOException(e);
1778     } catch (ExecutionException e) {
1779       if (e.getCause() instanceof IOException) {
1780         throw (IOException)e.getCause();
1781       } else {
1782         throw new IOException(e.getCause());
1783       }
1784     }
1785   }
1786 
1787   @Override
1788   public void createNamespace(final NamespaceDescriptor descriptor)
1789   throws IOException {
1790     get(createNamespaceAsync(descriptor), this.syncWaitTimeout, TimeUnit.MILLISECONDS);
1791   }
1792 
1793   @Override
1794   public Future<Void> createNamespaceAsync(final NamespaceDescriptor descriptor)
1795   throws IOException {
1796     CreateNamespaceResponse response =
1797         executeCallable(new MasterCallable<CreateNamespaceResponse>(getConnection()) {
1798       @Override
1799       public CreateNamespaceResponse call(int callTimeout) throws Exception {
1800         return master.createNamespace(null,
1801           CreateNamespaceRequest.newBuilder()
1802             .setNamespaceDescriptor(ProtobufUtil
1803               .toProtoNamespaceDescriptor(descriptor)).build()
1804         );
1805       }
1806     });
1807     return new NamespaceFuture(this, descriptor.getName(), response.getProcId()) {
1808       @Override
1809       public String getOperationType() {
1810         return "CREATE_NAMESPACE";
1811       }
1812     };
1813   }
1814 
1815   @Override
1816   public void modifyNamespace(final NamespaceDescriptor descriptor)
1817   throws IOException {
1818     get(modifyNamespaceAsync(descriptor), this.syncWaitTimeout, TimeUnit.MILLISECONDS);
1819   }
1820 
1821   @Override
1822   public Future<Void> modifyNamespaceAsync(final NamespaceDescriptor descriptor)
1823   throws IOException {
1824     ModifyNamespaceResponse response =
1825         executeCallable(new MasterCallable<ModifyNamespaceResponse>(getConnection()) {
1826       @Override
1827       public ModifyNamespaceResponse call(int callTimeout) throws Exception {
1828         return master.modifyNamespace(null, ModifyNamespaceRequest.newBuilder().
1829           setNamespaceDescriptor(ProtobufUtil.toProtoNamespaceDescriptor(descriptor)).build());
1830       }
1831     });
1832     return new NamespaceFuture(this, descriptor.getName(), response.getProcId()) {
1833       @Override
1834       public String getOperationType() {
1835         return "MODIFY_NAMESPACE";
1836       }
1837     };
1838   }
1839 
1840   @Override
1841   public void deleteNamespace(final String name)
1842   throws IOException {
1843     get(deleteNamespaceAsync(name), this.syncWaitTimeout, TimeUnit.MILLISECONDS);
1844   }
1845 
1846   @Override
1847   public Future<Void> deleteNamespaceAsync(final String name)
1848   throws IOException {
1849     DeleteNamespaceResponse response =
1850         executeCallable(new MasterCallable<DeleteNamespaceResponse>(getConnection()) {
1851       @Override
1852       public DeleteNamespaceResponse call(int callTimeout) throws Exception {
1853         return master.deleteNamespace(null, DeleteNamespaceRequest.newBuilder().
1854           setNamespaceName(name).build());
1855       }
1856     });
1857     return new NamespaceFuture(this, name, response.getProcId()) {
1858       @Override
1859       public String getOperationType() {
1860         return "DELETE_NAMESPACE";
1861       }
1862     };
1863   }
1864 
1865   @Override
1866   public NamespaceDescriptor getNamespaceDescriptor(final String name) throws IOException {
1867     return
1868         executeCallable(new MasterCallable<NamespaceDescriptor>(getConnection()) {
1869           @Override
1870           public NamespaceDescriptor call(int callTimeout) throws Exception {
1871             return ProtobufUtil.toNamespaceDescriptor(
1872               master.getNamespaceDescriptor(null, GetNamespaceDescriptorRequest.newBuilder().
1873                 setNamespaceName(name).build()).getNamespaceDescriptor());
1874           }
1875         });
1876   }
1877 
1878   @Override
1879   public NamespaceDescriptor[] listNamespaceDescriptors() throws IOException {
1880     return
1881         executeCallable(new MasterCallable<NamespaceDescriptor[]>(getConnection()) {
1882           @Override
1883           public NamespaceDescriptor[] call(int callTimeout) throws Exception {
1884             List<HBaseProtos.NamespaceDescriptor> list =
1885               master.listNamespaceDescriptors(null, ListNamespaceDescriptorsRequest.newBuilder().
1886                 build()).getNamespaceDescriptorList();
1887             NamespaceDescriptor[] res = new NamespaceDescriptor[list.size()];
1888             for(int i = 0; i < list.size(); i++) {
1889               res[i] = ProtobufUtil.toNamespaceDescriptor(list.get(i));
1890             }
1891             return res;
1892           }
1893         });
1894   }
1895 
1896   @Override
1897   public ProcedureInfo[] listProcedures() throws IOException {
1898     return
1899         executeCallable(new MasterCallable<ProcedureInfo[]>(getConnection()) {
1900           @Override
1901           public ProcedureInfo[] call(int callTimeout) throws Exception {
1902             List<ProcedureProtos.Procedure> procList = master.listProcedures(
1903               null, ListProceduresRequest.newBuilder().build()).getProcedureList();
1904             ProcedureInfo[] procInfoList = new ProcedureInfo[procList.size()];
1905             for (int i = 0; i < procList.size(); i++) {
1906               procInfoList[i] = ProcedureInfo.convert(procList.get(i));
1907             }
1908             return procInfoList;
1909           }
1910         });
1911   }
1912 
1913   @Override
1914   public HTableDescriptor[] listTableDescriptorsByNamespace(final String name) throws IOException {
1915     return
1916         executeCallable(new MasterCallable<HTableDescriptor[]>(getConnection()) {
1917           @Override
1918           public HTableDescriptor[] call(int callTimeout) throws Exception {
1919             List<TableSchema> list =
1920               master.listTableDescriptorsByNamespace(null, ListTableDescriptorsByNamespaceRequest.
1921                 newBuilder().setNamespaceName(name).build()).getTableSchemaList();
1922             HTableDescriptor[] res = new HTableDescriptor[list.size()];
1923             for(int i=0; i < list.size(); i++) {
1924 
1925               res[i] = HTableDescriptor.convert(list.get(i));
1926             }
1927             return res;
1928           }
1929         });
1930   }
1931 
1932   @Override
1933   public TableName[] listTableNamesByNamespace(final String name) throws IOException {
1934     return
1935         executeCallable(new MasterCallable<TableName[]>(getConnection()) {
1936           @Override
1937           public TableName[] call(int callTimeout) throws Exception {
1938             List<HBaseProtos.TableName> tableNames =
1939               master.listTableNamesByNamespace(null, ListTableNamesByNamespaceRequest.
1940                 newBuilder().setNamespaceName(name).build())
1941                 .getTableNameList();
1942             TableName[] result = new TableName[tableNames.size()];
1943             for (int i = 0; i < tableNames.size(); i++) {
1944               result[i] = ProtobufUtil.toTableName(tableNames.get(i));
1945             }
1946             return result;
1947           }
1948         });
1949   }
1950 
1951   /**
1952    * Check to see if HBase is running. Throw an exception if not.
1953    * @param conf system configuration
1954    * @throws MasterNotRunningException if the master is not running
1955    * @throws ZooKeeperConnectionException if unable to connect to zookeeper
1956    */
1957   // Used by tests and by the Merge tool. Merge tool uses it to figure if HBase is up or not.
1958   public static void checkHBaseAvailable(Configuration conf)
1959   throws MasterNotRunningException, ZooKeeperConnectionException, ServiceException, IOException {
1960     Configuration copyOfConf = HBaseConfiguration.create(conf);
1961     // We set it to make it fail as soon as possible if HBase is not available
1962     copyOfConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
1963     copyOfConf.setInt("zookeeper.recovery.retry", 0);
1964     try (ClusterConnection connection =
1965         (ClusterConnection)ConnectionFactory.createConnection(copyOfConf)) {
1966         // Check ZK first.
1967         // If the connection exists, we may have a connection to ZK that does not work anymore
1968         ZooKeeperKeepAliveConnection zkw = null;
1969         try {
1970           // This is NASTY. FIX!!!! Dependent on internal implementation! TODO
1971           zkw = ((ConnectionImplementation)connection).
1972             getKeepAliveZooKeeperWatcher();
1973           zkw.getRecoverableZooKeeper().getZooKeeper().exists(zkw.baseZNode, false);
1974         } catch (IOException e) {
1975           throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
1976         } catch (InterruptedException e) {
1977           throw (InterruptedIOException)
1978             new InterruptedIOException("Can't connect to ZooKeeper").initCause(e);
1979         } catch (KeeperException e) {
1980           throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
1981         } finally {
1982           if (zkw != null) {
1983             zkw.close();
1984           }
1985         }
1986       connection.isMasterRunning();
1987     }
1988   }
1989 
1990   @Override
1991   public List<HRegionInfo> getTableRegions(final TableName tableName)
1992   throws IOException {
1993     ZooKeeperWatcher zookeeper =
1994       new ZooKeeperWatcher(conf, ZK_IDENTIFIER_PREFIX + connection.toString(),
1995         new ThrowableAbortable());
1996     List<HRegionInfo> regions = null;
1997     try {
1998       if (TableName.META_TABLE_NAME.equals(tableName)) {
1999         regions = new MetaTableLocator().getMetaRegions(zookeeper);
2000       } else {
2001         regions = MetaTableAccessor.getTableRegions(connection, tableName, true);
2002       }
2003     } finally {
2004       zookeeper.close();
2005     }
2006     return regions;
2007   }
2008 
2009   @Override
2010   public synchronized void close() throws IOException {
2011   }
2012 
2013   @Override
2014   public HTableDescriptor[] getTableDescriptorsByTableName(final List<TableName> tableNames)
2015   throws IOException {
2016     return executeCallable(new MasterCallable<HTableDescriptor[]>(getConnection()) {
2017       @Override
2018       public HTableDescriptor[] call(int callTimeout) throws Exception {
2019         GetTableDescriptorsRequest req =
2020             RequestConverter.buildGetTableDescriptorsRequest(tableNames);
2021           return ProtobufUtil.getHTableDescriptorArray(master.getTableDescriptors(null, req));
2022       }
2023     });
2024   }
2025 
2026   /**
2027    * Get tableDescriptor
2028    * @param tableName one table name
2029    * @return HTD the HTableDescriptor or null if the table not exists
2030    * @throws IOException if a remote or network exception occurs
2031    */
2032   private HTableDescriptor getTableDescriptorByTableName(TableName tableName)
2033       throws IOException {
2034     List<TableName> tableNames = new ArrayList<TableName>(1);
2035     tableNames.add(tableName);
2036 
2037     HTableDescriptor[] htdl = getTableDescriptorsByTableName(tableNames);
2038 
2039     if (htdl == null || htdl.length == 0) {
2040       return null;
2041     }
2042     else {
2043       return htdl[0];
2044     }
2045   }
2046 
2047   @Override
2048   public HTableDescriptor[] getTableDescriptors(List<String> names)
2049   throws IOException {
2050     List<TableName> tableNames = new ArrayList<TableName>(names.size());
2051     for(String name : names) {
2052       tableNames.add(TableName.valueOf(name));
2053     }
2054     return getTableDescriptorsByTableName(tableNames);
2055   }
2056 
2057   private RollWALWriterResponse rollWALWriterImpl(final ServerName sn) throws IOException,
2058       FailedLogCloseException {
2059     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2060     RollWALWriterRequest request = RequestConverter.buildRollWALWriterRequest();
2061     try {
2062       return admin.rollWALWriter(null, request);
2063     } catch (ServiceException se) {
2064       throw ProtobufUtil.getRemoteException(se);
2065     }
2066   }
2067 
2068   /**
2069    * Roll the log writer. I.e. when using a file system based write ahead log,
2070    * start writing log messages to a new file.
2071    *
2072    * Note that when talking to a version 1.0+ HBase deployment, the rolling is asynchronous.
2073    * This method will return as soon as the roll is requested and the return value will
2074    * always be null. Additionally, the named region server may schedule store flushes at the
2075    * request of the wal handling the roll request.
2076    *
2077    * When talking to a 0.98 or older HBase deployment, the rolling is synchronous and the
2078    * return value may be either null or a list of encoded region names.
2079    *
2080    * @param serverName
2081    *          The servername of the regionserver. A server name is made of host,
2082    *          port and startcode. This is mandatory. Here is an example:
2083    *          <code> host187.example.com,60020,1289493121758</code>
2084    * @return a set of {@link HRegionInfo#getEncodedName()} that would allow the wal to
2085    *         clean up some underlying files. null if there's nothing to flush.
2086    * @throws IOException if a remote or network exception occurs
2087    * @throws FailedLogCloseException
2088    * @deprecated use {@link #rollWALWriter(ServerName)}
2089    */
2090   @Deprecated
2091   public synchronized byte[][] rollHLogWriter(String serverName)
2092       throws IOException, FailedLogCloseException {
2093     ServerName sn = ServerName.valueOf(serverName);
2094     final RollWALWriterResponse response = rollWALWriterImpl(sn);
2095     int regionCount = response.getRegionToFlushCount();
2096     if (0 == regionCount) {
2097       return null;
2098     }
2099     byte[][] regionsToFlush = new byte[regionCount][];
2100     for (int i = 0; i < regionCount; i++) {
2101       ByteString region = response.getRegionToFlush(i);
2102       regionsToFlush[i] = region.toByteArray();
2103     }
2104     return regionsToFlush;
2105   }
2106 
2107   @Override
2108   public synchronized void rollWALWriter(ServerName serverName)
2109       throws IOException, FailedLogCloseException {
2110     rollWALWriterImpl(serverName);
2111   }
2112 
2113   @Override
2114   public String[] getMasterCoprocessors() {
2115     try {
2116       return getClusterStatus().getMasterCoprocessors();
2117     } catch (IOException e) {
2118       LOG.error("Could not getClusterStatus()",e);
2119       return null;
2120     }
2121   }
2122 
2123   @Override
2124   public CompactionState getCompactionState(final TableName tableName)
2125   throws IOException {
2126     return getCompactionState(tableName, CompactType.NORMAL);
2127   }
2128 
2129   @Override
2130   public CompactionState getCompactionStateForRegion(final byte[] regionName)
2131   throws IOException {
2132     try {
2133       Pair<HRegionInfo, ServerName> regionServerPair = getRegion(regionName);
2134       if (regionServerPair == null) {
2135         throw new IllegalArgumentException("Invalid region: " + Bytes.toStringBinary(regionName));
2136       }
2137       if (regionServerPair.getSecond() == null) {
2138         throw new NoServerForRegionException(Bytes.toStringBinary(regionName));
2139       }
2140       ServerName sn = regionServerPair.getSecond();
2141       AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2142       GetRegionInfoRequest request = RequestConverter.buildGetRegionInfoRequest(
2143         regionServerPair.getFirst().getRegionName(), true);
2144       GetRegionInfoResponse response = admin.getRegionInfo(null, request);
2145       return response.getCompactionState();
2146     } catch (ServiceException se) {
2147       throw ProtobufUtil.getRemoteException(se);
2148     }
2149   }
2150 
2151   @Override
2152   public void snapshot(final String snapshotName,
2153                        final TableName tableName) throws IOException,
2154       SnapshotCreationException, IllegalArgumentException {
2155     snapshot(snapshotName, tableName, SnapshotDescription.Type.FLUSH);
2156   }
2157 
2158   @Override
2159   public void snapshot(final byte[] snapshotName, final TableName tableName)
2160       throws IOException, SnapshotCreationException, IllegalArgumentException {
2161     snapshot(Bytes.toString(snapshotName), tableName, SnapshotDescription.Type.FLUSH);
2162   }
2163 
2164   @Override
2165   public void snapshot(final String snapshotName, final TableName tableName,
2166       SnapshotDescription.Type type)
2167       throws IOException, SnapshotCreationException, IllegalArgumentException {
2168     SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
2169     builder.setTable(tableName.getNameAsString());
2170     builder.setName(snapshotName);
2171     builder.setType(type);
2172     snapshot(builder.build());
2173   }
2174 
2175   @Override
2176   public void snapshot(SnapshotDescription snapshot) throws IOException, SnapshotCreationException,
2177       IllegalArgumentException {
2178     // actually take the snapshot
2179     SnapshotResponse response = takeSnapshotAsync(snapshot);
2180     final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot)
2181         .build();
2182     IsSnapshotDoneResponse done = null;
2183     long start = EnvironmentEdgeManager.currentTime();
2184     long max = response.getExpectedTimeout();
2185     long maxPauseTime = max / this.numRetries;
2186     int tries = 0;
2187     LOG.debug("Waiting a max of " + max + " ms for snapshot '" +
2188         ClientSnapshotDescriptionUtils.toString(snapshot) + "'' to complete. (max " +
2189         maxPauseTime + " ms per retry)");
2190     while (tries == 0
2191         || ((EnvironmentEdgeManager.currentTime() - start) < max && !done.getDone())) {
2192       try {
2193         // sleep a backoff <= pauseTime amount
2194         long sleep = getPauseTime(tries++);
2195         sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
2196         LOG.debug("(#" + tries + ") Sleeping: " + sleep +
2197           "ms while waiting for snapshot completion.");
2198         Thread.sleep(sleep);
2199       } catch (InterruptedException e) {
2200         throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
2201       }
2202       LOG.debug("Getting current status of snapshot from master...");
2203       done = executeCallable(new MasterCallable<IsSnapshotDoneResponse>(getConnection()) {
2204         @Override
2205         public IsSnapshotDoneResponse call(int callTimeout) throws ServiceException {
2206           return master.isSnapshotDone(null, request);
2207         }
2208       });
2209     }
2210     if (!done.getDone()) {
2211       throw new SnapshotCreationException("Snapshot '" + snapshot.getName()
2212           + "' wasn't completed in expectedTime:" + max + " ms", snapshot);
2213     }
2214   }
2215 
2216   @Override
2217   public SnapshotResponse takeSnapshotAsync(SnapshotDescription snapshot) throws IOException,
2218       SnapshotCreationException {
2219     ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);
2220     final SnapshotRequest request = SnapshotRequest.newBuilder().setSnapshot(snapshot)
2221         .build();
2222     // run the snapshot on the master
2223     return executeCallable(new MasterCallable<SnapshotResponse>(getConnection()) {
2224       @Override
2225       public SnapshotResponse call(int callTimeout) throws ServiceException {
2226         return master.snapshot(null, request);
2227       }
2228     });
2229   }
2230 
2231   @Override
2232   public boolean isSnapshotFinished(final SnapshotDescription snapshot)
2233       throws IOException, HBaseSnapshotException, UnknownSnapshotException {
2234 
2235     return executeCallable(new MasterCallable<IsSnapshotDoneResponse>(getConnection()) {
2236       @Override
2237       public IsSnapshotDoneResponse call(int callTimeout) throws ServiceException {
2238         return master.isSnapshotDone(null,
2239           IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot).build());
2240       }
2241     }).getDone();
2242   }
2243 
2244   @Override
2245   public void restoreSnapshot(final byte[] snapshotName)
2246       throws IOException, RestoreSnapshotException {
2247     restoreSnapshot(Bytes.toString(snapshotName));
2248   }
2249 
2250   @Override
2251   public void restoreSnapshot(final String snapshotName)
2252       throws IOException, RestoreSnapshotException {
2253     boolean takeFailSafeSnapshot =
2254       conf.getBoolean("hbase.snapshot.restore.take.failsafe.snapshot", false);
2255     restoreSnapshot(snapshotName, takeFailSafeSnapshot);
2256   }
2257 
2258   @Override
2259   public void restoreSnapshot(final byte[] snapshotName, final boolean takeFailSafeSnapshot)
2260       throws IOException, RestoreSnapshotException {
2261     restoreSnapshot(Bytes.toString(snapshotName), takeFailSafeSnapshot);
2262   }
2263 
2264   @Override
2265   public void restoreSnapshot(final String snapshotName, boolean takeFailSafeSnapshot)
2266       throws IOException, RestoreSnapshotException {
2267     TableName tableName = null;
2268     for (SnapshotDescription snapshotInfo: listSnapshots()) {
2269       if (snapshotInfo.getName().equals(snapshotName)) {
2270         tableName = TableName.valueOf(snapshotInfo.getTable());
2271         break;
2272       }
2273     }
2274 
2275     if (tableName == null) {
2276       throw new RestoreSnapshotException(
2277         "Unable to find the table name for snapshot=" + snapshotName);
2278     }
2279 
2280     // The table does not exists, switch to clone.
2281     if (!tableExists(tableName)) {
2282       cloneSnapshot(snapshotName, tableName);
2283       return;
2284     }
2285 
2286     // Check if the table is disabled
2287     if (!isTableDisabled(tableName)) {
2288       throw new TableNotDisabledException(tableName);
2289     }
2290 
2291     // Take a snapshot of the current state
2292     String failSafeSnapshotSnapshotName = null;
2293     if (takeFailSafeSnapshot) {
2294       failSafeSnapshotSnapshotName = conf.get("hbase.snapshot.restore.failsafe.name",
2295         "hbase-failsafe-{snapshot.name}-{restore.timestamp}");
2296       failSafeSnapshotSnapshotName = failSafeSnapshotSnapshotName
2297         .replace("{snapshot.name}", snapshotName)
2298         .replace("{table.name}", tableName.toString().replace(TableName.NAMESPACE_DELIM, '.'))
2299         .replace("{restore.timestamp}", String.valueOf(EnvironmentEdgeManager.currentTime()));
2300       LOG.info("Taking restore-failsafe snapshot: " + failSafeSnapshotSnapshotName);
2301       snapshot(failSafeSnapshotSnapshotName, tableName);
2302     }
2303 
2304     try {
2305       // Restore snapshot
2306       internalRestoreSnapshot(snapshotName, tableName);
2307     } catch (IOException e) {
2308       // Somthing went wrong during the restore...
2309       // if the pre-restore snapshot is available try to rollback
2310       if (takeFailSafeSnapshot) {
2311         try {
2312           internalRestoreSnapshot(failSafeSnapshotSnapshotName, tableName);
2313           String msg = "Restore snapshot=" + snapshotName +
2314             " failed. Rollback to snapshot=" + failSafeSnapshotSnapshotName + " succeeded.";
2315           LOG.error(msg, e);
2316           throw new RestoreSnapshotException(msg, e);
2317         } catch (IOException ex) {
2318           String msg = "Failed to restore and rollback to snapshot=" + failSafeSnapshotSnapshotName;
2319           LOG.error(msg, ex);
2320           throw new RestoreSnapshotException(msg, e);
2321         }
2322       } else {
2323         throw new RestoreSnapshotException("Failed to restore snapshot=" + snapshotName, e);
2324       }
2325     }
2326 
2327     // If the restore is succeeded, delete the pre-restore snapshot
2328     if (takeFailSafeSnapshot) {
2329       try {
2330         LOG.info("Deleting restore-failsafe snapshot: " + failSafeSnapshotSnapshotName);
2331         deleteSnapshot(failSafeSnapshotSnapshotName);
2332       } catch (IOException e) {
2333         LOG.error("Unable to remove the failsafe snapshot: " + failSafeSnapshotSnapshotName, e);
2334       }
2335     }
2336   }
2337 
2338   @Override
2339   public void cloneSnapshot(final byte[] snapshotName, final TableName tableName)
2340       throws IOException, TableExistsException, RestoreSnapshotException {
2341     cloneSnapshot(Bytes.toString(snapshotName), tableName);
2342   }
2343 
2344   @Override
2345   public void cloneSnapshot(final String snapshotName, final TableName tableName)
2346       throws IOException, TableExistsException, RestoreSnapshotException {
2347     if (tableExists(tableName)) {
2348       throw new TableExistsException(tableName);
2349     }
2350     internalRestoreSnapshot(snapshotName, tableName);
2351     waitUntilTableIsEnabled(tableName);
2352   }
2353 
2354   @Override
2355   public byte[] execProcedureWithRet(String signature, String instance, Map<String, String> props)
2356       throws IOException {
2357     ProcedureDescription.Builder builder = ProcedureDescription.newBuilder();
2358     builder.setSignature(signature).setInstance(instance);
2359     for (Entry<String, String> entry : props.entrySet()) {
2360       NameStringPair pair = NameStringPair.newBuilder().setName(entry.getKey())
2361           .setValue(entry.getValue()).build();
2362       builder.addConfiguration(pair);
2363     }
2364 
2365     final ExecProcedureRequest request = ExecProcedureRequest.newBuilder()
2366         .setProcedure(builder.build()).build();
2367     // run the procedure on the master
2368     ExecProcedureResponse response = executeCallable(new MasterCallable<ExecProcedureResponse>(
2369         getConnection()) {
2370       @Override
2371       public ExecProcedureResponse call(int callTimeout) throws ServiceException {
2372         return master.execProcedureWithRet(null, request);
2373       }
2374     });
2375 
2376     return response.hasReturnData() ? response.getReturnData().toByteArray() : null;
2377   }
2378 
2379   @Override
2380   public void execProcedure(String signature, String instance, Map<String, String> props)
2381       throws IOException {
2382     ProcedureDescription.Builder builder = ProcedureDescription.newBuilder();
2383     builder.setSignature(signature).setInstance(instance);
2384     for (Entry<String, String> entry : props.entrySet()) {
2385       NameStringPair pair = NameStringPair.newBuilder().setName(entry.getKey())
2386           .setValue(entry.getValue()).build();
2387       builder.addConfiguration(pair);
2388     }
2389 
2390     final ExecProcedureRequest request = ExecProcedureRequest.newBuilder()
2391         .setProcedure(builder.build()).build();
2392     // run the procedure on the master
2393     ExecProcedureResponse response = executeCallable(new MasterCallable<ExecProcedureResponse>(
2394         getConnection()) {
2395       @Override
2396       public ExecProcedureResponse call(int callTimeout) throws ServiceException {
2397         return master.execProcedure(null, request);
2398       }
2399     });
2400 
2401     long start = EnvironmentEdgeManager.currentTime();
2402     long max = response.getExpectedTimeout();
2403     long maxPauseTime = max / this.numRetries;
2404     int tries = 0;
2405     LOG.debug("Waiting a max of " + max + " ms for procedure '" +
2406         signature + " : " + instance + "'' to complete. (max " + maxPauseTime + " ms per retry)");
2407     boolean done = false;
2408     while (tries == 0
2409         || ((EnvironmentEdgeManager.currentTime() - start) < max && !done)) {
2410       try {
2411         // sleep a backoff <= pauseTime amount
2412         long sleep = getPauseTime(tries++);
2413         sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
2414         LOG.debug("(#" + tries + ") Sleeping: " + sleep +
2415           "ms while waiting for procedure completion.");
2416         Thread.sleep(sleep);
2417       } catch (InterruptedException e) {
2418         throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
2419       }
2420       LOG.debug("Getting current status of procedure from master...");
2421       done = isProcedureFinished(signature, instance, props);
2422     }
2423     if (!done) {
2424       throw new IOException("Procedure '" + signature + " : " + instance
2425           + "' wasn't completed in expectedTime:" + max + " ms");
2426     }
2427   }
2428 
2429   @Override
2430   public boolean isProcedureFinished(String signature, String instance, Map<String, String> props)
2431       throws IOException {
2432     final ProcedureDescription.Builder builder = ProcedureDescription.newBuilder();
2433     builder.setSignature(signature).setInstance(instance);
2434     for (Entry<String, String> entry : props.entrySet()) {
2435       NameStringPair pair = NameStringPair.newBuilder().setName(entry.getKey())
2436           .setValue(entry.getValue()).build();
2437       builder.addConfiguration(pair);
2438     }
2439     final ProcedureDescription desc = builder.build();
2440     return executeCallable(
2441         new MasterCallable<IsProcedureDoneResponse>(getConnection()) {
2442           @Override
2443           public IsProcedureDoneResponse call(int callTimeout) throws ServiceException {
2444             return master.isProcedureDone(null, IsProcedureDoneRequest
2445                 .newBuilder().setProcedure(desc).build());
2446           }
2447         }).getDone();
2448   }
2449 
2450   /**
2451    * Execute Restore/Clone snapshot and wait for the server to complete (blocking).
2452    * To check if the cloned table exists, use {@link #isTableAvailable} -- it is not safe to
2453    * create an HTable instance to this table before it is available.
2454    * @param snapshotName snapshot to restore
2455    * @param tableName table name to restore the snapshot on
2456    * @throws IOException if a remote or network exception occurs
2457    * @throws RestoreSnapshotException if snapshot failed to be restored
2458    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2459    */
2460   private void internalRestoreSnapshot(final String snapshotName, final TableName tableName)
2461       throws IOException, RestoreSnapshotException {
2462     SnapshotDescription snapshot = SnapshotDescription.newBuilder()
2463         .setName(snapshotName).setTable(tableName.getNameAsString()).build();
2464 
2465     // actually restore the snapshot
2466     internalRestoreSnapshotAsync(snapshot);
2467 
2468     final IsRestoreSnapshotDoneRequest request = IsRestoreSnapshotDoneRequest.newBuilder()
2469         .setSnapshot(snapshot).build();
2470     IsRestoreSnapshotDoneResponse done = IsRestoreSnapshotDoneResponse.newBuilder()
2471         .setDone(false).buildPartial();
2472     final long maxPauseTime = 5000;
2473     int tries = 0;
2474     while (!done.getDone()) {
2475       try {
2476         // sleep a backoff <= pauseTime amount
2477         long sleep = getPauseTime(tries++);
2478         sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
2479         LOG.debug(tries + ") Sleeping: " + sleep
2480             + " ms while we wait for snapshot restore to complete.");
2481         Thread.sleep(sleep);
2482       } catch (InterruptedException e) {
2483         throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
2484       }
2485       LOG.debug("Getting current status of snapshot restore from master...");
2486       done = executeCallable(new MasterCallable<IsRestoreSnapshotDoneResponse>(
2487           getConnection()) {
2488         @Override
2489         public IsRestoreSnapshotDoneResponse call(int callTimeout) throws ServiceException {
2490           return master.isRestoreSnapshotDone(null, request);
2491         }
2492       });
2493     }
2494     if (!done.getDone()) {
2495       throw new RestoreSnapshotException("Snapshot '" + snapshot.getName() + "' wasn't restored.");
2496     }
2497   }
2498 
2499   /**
2500    * Execute Restore/Clone snapshot and wait for the server to complete (asynchronous)
2501    * <p>
2502    * Only a single snapshot should be restored at a time, or results may be undefined.
2503    * @param snapshot snapshot to restore
2504    * @return response from the server indicating the max time to wait for the snapshot
2505    * @throws IOException if a remote or network exception occurs
2506    * @throws RestoreSnapshotException if snapshot failed to be restored
2507    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2508    */
2509   private RestoreSnapshotResponse internalRestoreSnapshotAsync(final SnapshotDescription snapshot)
2510       throws IOException, RestoreSnapshotException {
2511     ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);
2512 
2513     final RestoreSnapshotRequest request = RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot)
2514         .build();
2515 
2516     // run the snapshot restore on the master
2517     return executeCallable(new MasterCallable<RestoreSnapshotResponse>(getConnection()) {
2518       @Override
2519       public RestoreSnapshotResponse call(int callTimeout) throws ServiceException {
2520         return master.restoreSnapshot(null, request);
2521       }
2522     });
2523   }
2524 
2525   @Override
2526   public List<SnapshotDescription> listSnapshots() throws IOException {
2527     return executeCallable(new MasterCallable<List<SnapshotDescription>>(getConnection()) {
2528       @Override
2529       public List<SnapshotDescription> call(int callTimeout) throws ServiceException {
2530         return master.getCompletedSnapshots(null, GetCompletedSnapshotsRequest.newBuilder().build())
2531             .getSnapshotsList();
2532       }
2533     });
2534   }
2535 
2536   @Override
2537   public List<SnapshotDescription> listSnapshots(String regex) throws IOException {
2538     return listSnapshots(Pattern.compile(regex));
2539   }
2540 
2541   @Override
2542   public List<SnapshotDescription> listSnapshots(Pattern pattern) throws IOException {
2543     List<SnapshotDescription> matched = new LinkedList<SnapshotDescription>();
2544     List<SnapshotDescription> snapshots = listSnapshots();
2545     for (SnapshotDescription snapshot : snapshots) {
2546       if (pattern.matcher(snapshot.getName()).matches()) {
2547         matched.add(snapshot);
2548       }
2549     }
2550     return matched;
2551   }
2552 
2553   @Override
2554   public List<SnapshotDescription> listTableSnapshots(String tableNameRegex,
2555       String snapshotNameRegex) throws IOException {
2556     return listTableSnapshots(Pattern.compile(tableNameRegex), Pattern.compile(snapshotNameRegex));
2557   }
2558 
2559   @Override
2560   public List<SnapshotDescription> listTableSnapshots(Pattern tableNamePattern,
2561       Pattern snapshotNamePattern) throws IOException {
2562     TableName[] tableNames = listTableNames(tableNamePattern);
2563 
2564     List<SnapshotDescription> tableSnapshots = new LinkedList<SnapshotDescription>();
2565     List<SnapshotDescription> snapshots = listSnapshots(snapshotNamePattern);
2566 
2567     List<TableName> listOfTableNames = Arrays.asList(tableNames);
2568     for (SnapshotDescription snapshot : snapshots) {
2569       if (listOfTableNames.contains(TableName.valueOf(snapshot.getTable()))) {
2570         tableSnapshots.add(snapshot);
2571       }
2572     }
2573     return tableSnapshots;
2574   }
2575 
2576   @Override
2577   public void deleteSnapshot(final byte[] snapshotName) throws IOException {
2578     deleteSnapshot(Bytes.toString(snapshotName));
2579   }
2580 
2581   @Override
2582   public void deleteSnapshot(final String snapshotName) throws IOException {
2583     // make sure the snapshot is possibly valid
2584     TableName.isLegalFullyQualifiedTableName(Bytes.toBytes(snapshotName));
2585     // do the delete
2586     executeCallable(new MasterCallable<Void>(getConnection()) {
2587       @Override
2588       public Void call(int callTimeout) throws ServiceException {
2589         master.deleteSnapshot(null,
2590           DeleteSnapshotRequest.newBuilder().
2591             setSnapshot(SnapshotDescription.newBuilder().setName(snapshotName).build()).build()
2592         );
2593         return null;
2594       }
2595     });
2596   }
2597 
2598   @Override
2599   public void deleteSnapshots(final String regex) throws IOException {
2600     deleteSnapshots(Pattern.compile(regex));
2601   }
2602 
2603   @Override
2604   public void deleteSnapshots(final Pattern pattern) throws IOException {
2605     List<SnapshotDescription> snapshots = listSnapshots(pattern);
2606     for (final SnapshotDescription snapshot : snapshots) {
2607       try {
2608         internalDeleteSnapshot(snapshot);
2609       } catch (IOException ex) {
2610         LOG.info(
2611           "Failed to delete snapshot " + snapshot.getName() + " for table " + snapshot.getTable(),
2612           ex);
2613       }
2614     }
2615   }
2616 
2617   private void internalDeleteSnapshot(final SnapshotDescription snapshot) throws IOException {
2618     executeCallable(new MasterCallable<Void>(getConnection()) {
2619       @Override
2620       public Void call(int callTimeout) throws ServiceException {
2621         this.master.deleteSnapshot(null, DeleteSnapshotRequest.newBuilder().setSnapshot(snapshot)
2622             .build());
2623         return null;
2624       }
2625     });
2626   }
2627 
2628   @Override
2629   public void deleteTableSnapshots(String tableNameRegex, String snapshotNameRegex)
2630       throws IOException {
2631     deleteTableSnapshots(Pattern.compile(tableNameRegex), Pattern.compile(snapshotNameRegex));
2632   }
2633 
2634   @Override
2635   public void deleteTableSnapshots(Pattern tableNamePattern, Pattern snapshotNamePattern)
2636       throws IOException {
2637     List<SnapshotDescription> snapshots = listTableSnapshots(tableNamePattern, snapshotNamePattern);
2638     for (SnapshotDescription snapshot : snapshots) {
2639       try {
2640         internalDeleteSnapshot(snapshot);
2641         LOG.debug("Successfully deleted snapshot: " + snapshot.getName());
2642       } catch (IOException e) {
2643         LOG.error("Failed to delete snapshot: " + snapshot.getName(), e);
2644       }
2645     }
2646   }
2647 
2648   @Override
2649   public void setQuota(final QuotaSettings quota) throws IOException {
2650     executeCallable(new MasterCallable<Void>(getConnection()) {
2651       @Override
2652       public Void call(int callTimeout) throws ServiceException {
2653         this.master.setQuota(null, QuotaSettings.buildSetQuotaRequestProto(quota));
2654         return null;
2655       }
2656     });
2657   }
2658 
2659   @Override
2660   public QuotaRetriever getQuotaRetriever(final QuotaFilter filter) throws IOException {
2661     return QuotaRetriever.open(conf, filter);
2662   }
2663 
2664   private <C extends RetryingCallable<V> & Closeable, V> V executeCallable(C callable)
2665       throws IOException {
2666     return executeCallable(callable, rpcCallerFactory, operationTimeout);
2667   }
2668 
2669   private static <C extends RetryingCallable<V> & Closeable, V> V executeCallable(C callable,
2670              RpcRetryingCallerFactory rpcCallerFactory, int operationTimeout) throws IOException {
2671     RpcRetryingCaller<V> caller = rpcCallerFactory.newCaller();
2672     try {
2673       return caller.callWithRetries(callable, operationTimeout);
2674     } finally {
2675       callable.close();
2676     }
2677   }
2678 
2679   @Override
2680   public CoprocessorRpcChannel coprocessorService() {
2681     return new MasterCoprocessorRpcChannel(connection);
2682   }
2683 
2684   /**
2685    * Simple {@link Abortable}, throwing RuntimeException on abort.
2686    */
2687   private static class ThrowableAbortable implements Abortable {
2688 
2689     @Override
2690     public void abort(String why, Throwable e) {
2691       throw new RuntimeException(why, e);
2692     }
2693 
2694     @Override
2695     public boolean isAborted() {
2696       return true;
2697     }
2698   }
2699 
2700   @Override
2701   public CoprocessorRpcChannel coprocessorService(ServerName sn) {
2702     return new RegionServerCoprocessorRpcChannel(connection, sn);
2703   }
2704 
2705   @Override
2706   public void updateConfiguration(ServerName server) throws IOException {
2707     try {
2708       this.connection.getAdmin(server).updateConfiguration(null,
2709         UpdateConfigurationRequest.getDefaultInstance());
2710     } catch (ServiceException e) {
2711       throw ProtobufUtil.getRemoteException(e);
2712     }
2713   }
2714 
2715   @Override
2716   public void updateConfiguration() throws IOException {
2717     for (ServerName server : this.getClusterStatus().getServers()) {
2718       updateConfiguration(server);
2719     }
2720   }
2721 
2722   @Override
2723   public int getMasterInfoPort() throws IOException {
2724     // TODO: Fix!  Reaching into internal implementation!!!!
2725     ConnectionImplementation connection =
2726         (ConnectionImplementation)this.connection;
2727     ZooKeeperKeepAliveConnection zkw = connection.getKeepAliveZooKeeperWatcher();
2728     try {
2729       return MasterAddressTracker.getMasterInfoPort(zkw);
2730     } catch (KeeperException e) {
2731       throw new IOException("Failed to get master info port from MasterAddressTracker", e);
2732     }
2733   }
2734 
2735   private ServerName getMasterAddress() throws IOException {
2736     // TODO: Fix!  Reaching into internal implementation!!!!
2737     ConnectionImplementation connection =
2738             (ConnectionImplementation)this.connection;
2739     ZooKeeperKeepAliveConnection zkw = connection.getKeepAliveZooKeeperWatcher();
2740     try {
2741       return MasterAddressTracker.getMasterAddress(zkw);
2742     } catch (KeeperException e) {
2743       throw new IOException("Failed to get master server name from MasterAddressTracker", e);
2744     }
2745   }
2746 
2747   @Override
2748   public long getLastMajorCompactionTimestamp(final TableName tableName) throws IOException {
2749     return executeCallable(new MasterCallable<Long>(getConnection()) {
2750       @Override
2751       public Long call(int callTimeout) throws ServiceException {
2752         MajorCompactionTimestampRequest req =
2753             MajorCompactionTimestampRequest.newBuilder()
2754                 .setTableName(ProtobufUtil.toProtoTableName(tableName)).build();
2755         return master.getLastMajorCompactionTimestamp(null, req).getCompactionTimestamp();
2756       }
2757     });
2758   }
2759 
2760   @Override
2761   public long getLastMajorCompactionTimestampForRegion(final byte[] regionName) throws IOException {
2762     return executeCallable(new MasterCallable<Long>(getConnection()) {
2763       @Override
2764       public Long call(int callTimeout) throws ServiceException {
2765         MajorCompactionTimestampForRegionRequest req =
2766             MajorCompactionTimestampForRegionRequest
2767                 .newBuilder()
2768                 .setRegion(
2769                   RequestConverter
2770                       .buildRegionSpecifier(RegionSpecifierType.REGION_NAME, regionName)).build();
2771         return master.getLastMajorCompactionTimestampForRegion(null, req).getCompactionTimestamp();
2772       }
2773     });
2774   }
2775 
2776   /**
2777    * {@inheritDoc}
2778    */
2779   @Override
2780   public void compact(final TableName tableName, final byte[] columnFamily, CompactType compactType)
2781     throws IOException, InterruptedException {
2782     compact(tableName, columnFamily, false, compactType);
2783   }
2784 
2785   /**
2786    * {@inheritDoc}
2787    */
2788   @Override
2789   public void compact(final TableName tableName, CompactType compactType)
2790     throws IOException, InterruptedException {
2791     compact(tableName, null, false, compactType);
2792   }
2793 
2794   /**
2795    * {@inheritDoc}
2796    */
2797   @Override
2798   public void majorCompact(final TableName tableName, final byte[] columnFamily,
2799     CompactType compactType) throws IOException, InterruptedException {
2800     compact(tableName, columnFamily, true, compactType);
2801   }
2802 
2803   /**
2804    * {@inheritDoc}
2805    */
2806   @Override
2807   public void majorCompact(final TableName tableName, CompactType compactType)
2808           throws IOException, InterruptedException {
2809       compact(tableName, null, true, compactType);
2810   }
2811 
2812   /**
2813    * {@inheritDoc}
2814    */
2815   @Override
2816   public CompactionState getCompactionState(TableName tableName,
2817     CompactType compactType) throws IOException {
2818     CompactionState state = CompactionState.NONE;
2819     checkTableExists(tableName);
2820     switch (compactType) {
2821       case MOB:
2822         try {
2823           ServerName master = getMasterAddress();
2824           HRegionInfo info = getMobRegionInfo(tableName);
2825           GetRegionInfoRequest request = RequestConverter.buildGetRegionInfoRequest(
2826                   info.getRegionName(), true);
2827           GetRegionInfoResponse response = this.connection.getAdmin(master)
2828                   .getRegionInfo(null, request);
2829           state = response.getCompactionState();
2830         } catch (ServiceException se) {
2831           throw ProtobufUtil.getRemoteException(se);
2832         }
2833         break;
2834       case NORMAL:
2835       default:
2836         ZooKeeperWatcher zookeeper =
2837                 new ZooKeeperWatcher(conf, ZK_IDENTIFIER_PREFIX + connection.toString(),
2838                         new ThrowableAbortable());
2839         try {
2840           List<Pair<HRegionInfo, ServerName>> pairs;
2841           if (TableName.META_TABLE_NAME.equals(tableName)) {
2842             pairs = new MetaTableLocator().getMetaRegionsAndLocations(zookeeper);
2843           } else {
2844             pairs = MetaTableAccessor.getTableRegionsAndLocations(connection, tableName);
2845           }
2846           for (Pair<HRegionInfo, ServerName> pair : pairs) {
2847             if (pair.getFirst().isOffline()) continue;
2848             if (pair.getSecond() == null) continue;
2849             try {
2850               ServerName sn = pair.getSecond();
2851               AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2852               GetRegionInfoRequest request = RequestConverter.buildGetRegionInfoRequest(
2853                       pair.getFirst().getRegionName(), true);
2854               GetRegionInfoResponse response = admin.getRegionInfo(null, request);
2855               switch (response.getCompactionState()) {
2856                 case MAJOR_AND_MINOR:
2857                   return CompactionState.MAJOR_AND_MINOR;
2858                 case MAJOR:
2859                   if (state == CompactionState.MINOR) {
2860                     return CompactionState.MAJOR_AND_MINOR;
2861                   }
2862                   state = CompactionState.MAJOR;
2863                   break;
2864                 case MINOR:
2865                   if (state == CompactionState.MAJOR) {
2866                     return CompactionState.MAJOR_AND_MINOR;
2867                   }
2868                   state = CompactionState.MINOR;
2869                   break;
2870                 case NONE:
2871                 default: // nothing, continue
2872               }
2873             } catch (NotServingRegionException e) {
2874               if (LOG.isDebugEnabled()) {
2875                 LOG.debug("Trying to get compaction state of " +
2876                         pair.getFirst() + ": " +
2877                         StringUtils.stringifyException(e));
2878               }
2879             } catch (RemoteException e) {
2880               if (e.getMessage().indexOf(NotServingRegionException.class.getName()) >= 0) {
2881                 if (LOG.isDebugEnabled()) {
2882                   LOG.debug("Trying to get compaction state of " + pair.getFirst() + ": "
2883                           + StringUtils.stringifyException(e));
2884                 }
2885               } else {
2886                 throw e;
2887               }
2888             }
2889           }
2890         } catch (ServiceException se) {
2891           throw ProtobufUtil.getRemoteException(se);
2892         } finally {
2893           zookeeper.close();
2894         }
2895         break;
2896     }
2897     return state;
2898   }
2899 
2900   /**
2901    * Future that waits on a procedure result.
2902    * Returned by the async version of the Admin calls,
2903    * and used internally by the sync calls to wait on the result of the procedure.
2904    */
2905   @InterfaceAudience.Private
2906   @InterfaceStability.Evolving
2907   protected static class ProcedureFuture<V> implements Future<V> {
2908     private ExecutionException exception = null;
2909     private boolean procResultFound = false;
2910     private boolean done = false;
2911     private boolean cancelled = false;
2912     private V result = null;
2913 
2914     private final HBaseAdmin admin;
2915     private final Long procId;
2916 
2917     public ProcedureFuture(final HBaseAdmin admin, final Long procId) {
2918       this.admin = admin;
2919       this.procId = procId;
2920     }
2921 
2922     @Override
2923     public boolean cancel(boolean mayInterruptIfRunning) {
2924       AbortProcedureRequest abortProcRequest = AbortProcedureRequest.newBuilder()
2925           .setProcId(procId).setMayInterruptIfRunning(mayInterruptIfRunning).build();
2926       try {
2927         cancelled = abortProcedureResult(abortProcRequest).getIsProcedureAborted();
2928         if (cancelled) {
2929           done = true;
2930         }
2931       } catch (IOException e) {
2932         // Cancell thrown exception for some reason. At this time, we are not sure whether
2933         // the cancell succeeds or fails. We assume that it is failed, but print out a warning
2934         // for debugging purpose.
2935         LOG.warn(
2936           "Cancelling the procedure with procId=" + procId + " throws exception " + e.getMessage(),
2937           e);
2938         cancelled = false;
2939       }
2940       return cancelled;
2941     }
2942 
2943     @Override
2944     public boolean isCancelled() {
2945       return cancelled;
2946     }
2947 
2948     protected AbortProcedureResponse abortProcedureResult(
2949         final AbortProcedureRequest request) throws IOException {
2950       return admin.executeCallable(new MasterCallable<AbortProcedureResponse>(
2951           admin.getConnection()) {
2952         @Override
2953         public AbortProcedureResponse call(int callTimeout) throws ServiceException {
2954           return master.abortProcedure(null, request);
2955         }
2956       });
2957     }
2958 
2959     @Override
2960     public V get() throws InterruptedException, ExecutionException {
2961       // TODO: should we ever spin forever?
2962       throw new UnsupportedOperationException();
2963     }
2964 
2965     @Override
2966     public V get(long timeout, TimeUnit unit)
2967         throws InterruptedException, ExecutionException, TimeoutException {
2968       if (!done) {
2969         long deadlineTs = EnvironmentEdgeManager.currentTime() + unit.toMillis(timeout);
2970         try {
2971           try {
2972             // if the master support procedures, try to wait the result
2973             if (procId != null) {
2974               result = waitProcedureResult(procId, deadlineTs);
2975             }
2976             // if we don't have a proc result, try the compatibility wait
2977             if (!procResultFound) {
2978               result = waitOperationResult(deadlineTs);
2979             }
2980             result = postOperationResult(result, deadlineTs);
2981             done = true;
2982           } catch (IOException e) {
2983             result = postOperationFailure(e, deadlineTs);
2984             done = true;
2985           }
2986         } catch (IOException e) {
2987           exception = new ExecutionException(e);
2988           done = true;
2989         }
2990       }
2991       if (exception != null) {
2992         throw exception;
2993       }
2994       return result;
2995     }
2996 
2997     @Override
2998     public boolean isDone() {
2999       return done;
3000     }
3001 
3002     protected HBaseAdmin getAdmin() {
3003       return admin;
3004     }
3005 
3006     private V waitProcedureResult(long procId, long deadlineTs)
3007         throws IOException, TimeoutException, InterruptedException {
3008       GetProcedureResultRequest request = GetProcedureResultRequest.newBuilder()
3009           .setProcId(procId)
3010           .build();
3011 
3012       int tries = 0;
3013       IOException serviceEx = null;
3014       while (EnvironmentEdgeManager.currentTime() < deadlineTs) {
3015         GetProcedureResultResponse response = null;
3016         try {
3017           // Try to fetch the result
3018           response = getProcedureResult(request);
3019         } catch (IOException e) {
3020           serviceEx = unwrapException(e);
3021 
3022           // the master may be down
3023           LOG.warn("failed to get the procedure result procId=" + procId, serviceEx);
3024 
3025           // Not much to do, if we have a DoNotRetryIOException
3026           if (serviceEx instanceof DoNotRetryIOException) {
3027             // TODO: looks like there is no way to unwrap this exception and get the proper
3028             // UnsupportedOperationException aside from looking at the message.
3029             // anyway, if we fail here we just failover to the compatibility side
3030             // and that is always a valid solution.
3031             LOG.warn("Proc-v2 is unsupported on this master: " + serviceEx.getMessage(), serviceEx);
3032             procResultFound = false;
3033             return null;
3034           }
3035         }
3036 
3037         // If the procedure is no longer running, we should have a result
3038         if (response != null && response.getState() != GetProcedureResultResponse.State.RUNNING) {
3039           procResultFound = response.getState() != GetProcedureResultResponse.State.NOT_FOUND;
3040           return convertResult(response);
3041         }
3042 
3043         try {
3044           Thread.sleep(getAdmin().getPauseTime(tries++));
3045         } catch (InterruptedException e) {
3046           throw new InterruptedException(
3047             "Interrupted while waiting for the result of proc " + procId);
3048         }
3049       }
3050       if (serviceEx != null) {
3051         throw serviceEx;
3052       } else {
3053         throw new TimeoutException("The procedure " + procId + " is still running");
3054       }
3055     }
3056 
3057     private static IOException unwrapException(IOException e) {
3058       if (e instanceof RemoteException) {
3059         return ((RemoteException)e).unwrapRemoteException();
3060       }
3061       return e;
3062     }
3063 
3064     protected GetProcedureResultResponse getProcedureResult(final GetProcedureResultRequest request)
3065         throws IOException {
3066       return admin.executeCallable(new MasterCallable<GetProcedureResultResponse>(
3067           admin.getConnection()) {
3068         @Override
3069         public GetProcedureResultResponse call(int callTimeout) throws ServiceException {
3070           return master.getProcedureResult(null, request);
3071         }
3072       });
3073     }
3074 
3075     /**
3076      * Convert the procedure result response to a specified type.
3077      * @param response the procedure result object to parse
3078      * @return the result data of the procedure.
3079      */
3080     protected V convertResult(final GetProcedureResultResponse response) throws IOException {
3081       if (response.hasException()) {
3082         throw ForeignExceptionUtil.toIOException(response.getException());
3083       }
3084       return null;
3085     }
3086 
3087     /**
3088      * Fallback implementation in case the procedure is not supported by the server.
3089      * It should try to wait until the operation is completed.
3090      * @param deadlineTs the timestamp after which this method should throw a TimeoutException
3091      * @return the result data of the operation
3092      */
3093     protected V waitOperationResult(final long deadlineTs)
3094         throws IOException, TimeoutException {
3095       return null;
3096     }
3097 
3098     /**
3099      * Called after the operation is completed and the result fetched. this allows to perform extra
3100      * steps after the procedure is completed. it allows to apply transformations to the result that
3101      * will be returned by get().
3102      * @param result the result of the procedure
3103      * @param deadlineTs the timestamp after which this method should throw a TimeoutException
3104      * @return the result of the procedure, which may be the same as the passed one
3105      */
3106     protected V postOperationResult(final V result, final long deadlineTs)
3107         throws IOException, TimeoutException {
3108       return result;
3109     }
3110 
3111     /**
3112      * Called after the operation is terminated with a failure.
3113      * this allows to perform extra steps after the procedure is terminated.
3114      * it allows to apply transformations to the result that will be returned by get().
3115      * The default implementation will rethrow the exception
3116      * @param exception the exception got from fetching the result
3117      * @param deadlineTs the timestamp after which this method should throw a TimeoutException
3118      * @return the result of the procedure, which may be the same as the passed one
3119      */
3120     protected V postOperationFailure(final IOException exception, final long deadlineTs)
3121         throws IOException, TimeoutException {
3122       throw exception;
3123     }
3124 
3125     protected interface WaitForStateCallable {
3126       boolean checkState(int tries) throws IOException;
3127       void throwInterruptedException() throws InterruptedIOException;
3128       void throwTimeoutException(long elapsed) throws TimeoutException;
3129     }
3130 
3131     protected void waitForState(final long deadlineTs, final WaitForStateCallable callable)
3132         throws IOException, TimeoutException {
3133       int tries = 0;
3134       IOException serverEx = null;
3135       long startTime = EnvironmentEdgeManager.currentTime();
3136       while (EnvironmentEdgeManager.currentTime() < deadlineTs) {
3137         serverEx = null;
3138         try {
3139           if (callable.checkState(tries)) {
3140             return;
3141           }
3142         } catch (IOException e) {
3143           serverEx = e;
3144         }
3145         try {
3146           Thread.sleep(getAdmin().getPauseTime(tries++));
3147         } catch (InterruptedException e) {
3148           callable.throwInterruptedException();
3149         }
3150       }
3151       if (serverEx != null) {
3152         throw unwrapException(serverEx);
3153       } else {
3154         callable.throwTimeoutException(EnvironmentEdgeManager.currentTime() - startTime);
3155       }
3156     }
3157   }
3158 
3159   @InterfaceAudience.Private
3160   @InterfaceStability.Evolving
3161   protected static abstract class TableFuture<V> extends ProcedureFuture<V> {
3162     private final TableName tableName;
3163 
3164     public TableFuture(final HBaseAdmin admin, final TableName tableName, final Long procId) {
3165       super(admin, procId);
3166       this.tableName = tableName;
3167     }
3168 
3169     @Override
3170     public String toString() {
3171       return getDescription();
3172     }
3173 
3174     /**
3175      * @return the table name
3176      */
3177     protected TableName getTableName() {
3178       return tableName;
3179     }
3180 
3181     /**
3182      * @return the table descriptor
3183      */
3184     protected HTableDescriptor getTableDescriptor() throws IOException {
3185       return getAdmin().getTableDescriptorByTableName(getTableName());
3186     }
3187 
3188     /**
3189      * @return the operation type like CREATE, DELETE, DISABLE etc.
3190      */
3191     public abstract String getOperationType();
3192 
3193     /**
3194      * @return a description of the operation
3195      */
3196     protected String getDescription() {
3197       return "Operation: " + getOperationType() + ", "
3198           + "Table Name: " + tableName.getNameWithNamespaceInclAsString();
3199 
3200     };
3201 
3202     protected abstract class TableWaitForStateCallable implements WaitForStateCallable {
3203       @Override
3204       public void throwInterruptedException() throws InterruptedIOException {
3205         throw new InterruptedIOException("Interrupted while waiting for operation: "
3206             + getOperationType() + " on table: " + tableName.getNameWithNamespaceInclAsString());
3207       }
3208 
3209       @Override
3210       public void throwTimeoutException(long elapsedTime) throws TimeoutException {
3211         throw new TimeoutException("The operation: " + getOperationType() + " on table: " +
3212             tableName.getNameAsString() + " has not completed after " + elapsedTime + "ms");
3213       }
3214     }
3215 
3216     @Override
3217     protected V postOperationResult(final V result, final long deadlineTs)
3218         throws IOException, TimeoutException {
3219       LOG.info(getDescription() + " completed");
3220       return super.postOperationResult(result, deadlineTs);
3221     }
3222 
3223     @Override
3224     protected V postOperationFailure(final IOException exception, final long deadlineTs)
3225         throws IOException, TimeoutException {
3226       LOG.info(getDescription() + " failed with " + exception.getMessage());
3227       return super.postOperationFailure(exception, deadlineTs);
3228     }
3229 
3230     protected void waitForTableEnabled(final long deadlineTs)
3231         throws IOException, TimeoutException {
3232       waitForState(deadlineTs, new TableWaitForStateCallable() {
3233         @Override
3234         public boolean checkState(int tries) throws IOException {
3235           try {
3236             if (getAdmin().isTableAvailable(tableName)) {
3237               return true;
3238             }
3239           } catch (TableNotFoundException tnfe) {
3240             LOG.debug("Table " + tableName.getNameWithNamespaceInclAsString()
3241                 + " was not enabled, sleeping. tries=" + tries);
3242           }
3243           return false;
3244         }
3245       });
3246     }
3247 
3248     protected void waitForTableDisabled(final long deadlineTs)
3249         throws IOException, TimeoutException {
3250       waitForState(deadlineTs, new TableWaitForStateCallable() {
3251         @Override
3252         public boolean checkState(int tries) throws IOException {
3253           return getAdmin().isTableDisabled(tableName);
3254         }
3255       });
3256     }
3257 
3258     protected void waitTableNotFound(final long deadlineTs)
3259         throws IOException, TimeoutException {
3260       waitForState(deadlineTs, new TableWaitForStateCallable() {
3261         @Override
3262         public boolean checkState(int tries) throws IOException {
3263           return !getAdmin().tableExists(tableName);
3264         }
3265       });
3266     }
3267 
3268     protected void waitForSchemaUpdate(final long deadlineTs)
3269         throws IOException, TimeoutException {
3270       waitForState(deadlineTs, new TableWaitForStateCallable() {
3271         @Override
3272         public boolean checkState(int tries) throws IOException {
3273           return getAdmin().getAlterStatus(tableName).getFirst() == 0;
3274         }
3275       });
3276     }
3277 
3278     protected void waitForAllRegionsOnline(final long deadlineTs, final byte[][] splitKeys)
3279         throws IOException, TimeoutException {
3280       final HTableDescriptor desc = getTableDescriptor();
3281       final AtomicInteger actualRegCount = new AtomicInteger(0);
3282       final MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {
3283         @Override
3284         public boolean visit(Result rowResult) throws IOException {
3285           RegionLocations list = MetaTableAccessor.getRegionLocations(rowResult);
3286           if (list == null) {
3287             LOG.warn("No serialized HRegionInfo in " + rowResult);
3288             return true;
3289           }
3290           HRegionLocation l = list.getRegionLocation();
3291           if (l == null) {
3292             return true;
3293           }
3294           if (!l.getRegionInfo().getTable().equals(desc.getTableName())) {
3295             return false;
3296           }
3297           if (l.getRegionInfo().isOffline() || l.getRegionInfo().isSplit()) return true;
3298           HRegionLocation[] locations = list.getRegionLocations();
3299           for (HRegionLocation location : locations) {
3300             if (location == null) continue;
3301             ServerName serverName = location.getServerName();
3302             // Make sure that regions are assigned to server
3303             if (serverName != null && serverName.getHostAndPort() != null) {
3304               actualRegCount.incrementAndGet();
3305             }
3306           }
3307           return true;
3308         }
3309       };
3310 
3311       int tries = 0;
3312       int numRegs = (splitKeys == null ? 1 : splitKeys.length + 1) * desc.getRegionReplication();
3313       while (EnvironmentEdgeManager.currentTime() < deadlineTs) {
3314         actualRegCount.set(0);
3315         MetaTableAccessor.scanMetaForTableRegions(getAdmin().getConnection(), visitor,
3316           desc.getTableName());
3317         if (actualRegCount.get() == numRegs) {
3318           // all the regions are online
3319           return;
3320         }
3321 
3322         try {
3323           Thread.sleep(getAdmin().getPauseTime(tries++));
3324         } catch (InterruptedException e) {
3325           throw new InterruptedIOException("Interrupted when opening" + " regions; "
3326               + actualRegCount.get() + " of " + numRegs + " regions processed so far");
3327         }
3328       }
3329       throw new TimeoutException("Only " + actualRegCount.get() + " of " + numRegs
3330           + " regions are online; retries exhausted.");
3331     }
3332   }
3333 
3334   @InterfaceAudience.Private
3335   @InterfaceStability.Evolving
3336   protected static abstract class NamespaceFuture extends ProcedureFuture<Void> {
3337     private final String namespaceName;
3338 
3339     public NamespaceFuture(final HBaseAdmin admin, final String namespaceName, final Long procId) {
3340       super(admin, procId);
3341       this.namespaceName = namespaceName;
3342     }
3343 
3344     /**
3345      * @return the namespace name
3346      */
3347     protected String getNamespaceName() {
3348       return namespaceName;
3349     }
3350 
3351     /**
3352      * @return the operation type like CREATE_NAMESPACE, DELETE_NAMESPACE, etc.
3353      */
3354     public abstract String getOperationType();
3355 
3356     @Override
3357     public String toString() {
3358       return "Operation: " + getOperationType() + ", Namespace: " + getNamespaceName();
3359     }
3360   }
3361 
3362   @Override
3363   public List<SecurityCapability> getSecurityCapabilities() throws IOException {
3364     try {
3365       return executeCallable(new MasterCallable<List<SecurityCapability>>(getConnection()) {
3366         @Override
3367         public List<SecurityCapability> call(int callTimeout) throws ServiceException {
3368           SecurityCapabilitiesRequest req = SecurityCapabilitiesRequest.newBuilder().build();
3369           return ProtobufUtil.toSecurityCapabilityList(
3370             master.getSecurityCapabilities(null, req).getCapabilitiesList());
3371         }
3372       });
3373     } catch (IOException e) {
3374       if (e instanceof RemoteException) {
3375         e = ((RemoteException)e).unwrapRemoteException();
3376       }
3377       throw e;
3378     }
3379   }
3380 
3381   private HRegionInfo getMobRegionInfo(TableName tableName) {
3382     return new HRegionInfo(tableName, Bytes.toBytes(".mob"),
3383             HConstants.EMPTY_END_ROW, false, 0);
3384   }
3385 
3386 }