001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static java.util.stream.Collectors.toList;
021import static org.apache.hadoop.hbase.client.ConnectionUtils.checkHasFamilies;
022import static org.apache.hadoop.hbase.client.ConnectionUtils.isEmptyStopRow;
023import static org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead;
024import static org.apache.hadoop.hbase.util.FutureUtils.addListener;
025
026import com.google.protobuf.RpcChannel;
027import java.io.IOException;
028import java.util.ArrayList;
029import java.util.Arrays;
030import java.util.List;
031import java.util.concurrent.CompletableFuture;
032import java.util.concurrent.TimeUnit;
033import java.util.concurrent.atomic.AtomicBoolean;
034import java.util.concurrent.atomic.AtomicInteger;
035import java.util.function.Function;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.CompareOperator;
038import org.apache.hadoop.hbase.HConstants;
039import org.apache.hadoop.hbase.HRegionLocation;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory.SingleRequestCallerBuilder;
042import org.apache.hadoop.hbase.filter.BinaryComparator;
043import org.apache.hadoop.hbase.io.TimeRange;
044import org.apache.hadoop.hbase.ipc.HBaseRpcController;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.apache.hadoop.hbase.util.ReflectionUtils;
047import org.apache.yetus.audience.InterfaceAudience;
048
049import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
050import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;
051import org.apache.hbase.thirdparty.io.netty.util.Timer;
052
053import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
054import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
055import org.apache.hadoop.hbase.shaded.protobuf.ResponseConverter;
056import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ClientService;
057import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.GetRequest;
058import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.GetResponse;
059import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MultiRequest;
060import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MultiResponse;
061import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutateRequest;
062import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutateResponse;
063import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.RegionAction;
064import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CompareType;
065
066/**
067 * The implementation of RawAsyncTable.
068 * <p/>
069 * The word 'Raw' means that this is a low level class. The returned {@link CompletableFuture} will
070 * be finished inside the rpc framework thread, which means that the callbacks registered to the
071 * {@link CompletableFuture} will also be executed inside the rpc framework thread. So users who use
072 * this class should not try to do time consuming tasks in the callbacks.
073 * @since 2.0.0
074 * @see AsyncTableImpl
075 */
076@InterfaceAudience.Private
077class RawAsyncTableImpl implements AsyncTable<AdvancedScanResultConsumer> {
078
079  private final AsyncConnectionImpl conn;
080
081  private final Timer retryTimer;
082
083  private final TableName tableName;
084
085  private final int defaultScannerCaching;
086
087  private final long defaultScannerMaxResultSize;
088
089  private final long rpcTimeoutNs;
090
091  private final long readRpcTimeoutNs;
092
093  private final long writeRpcTimeoutNs;
094
095  private final long operationTimeoutNs;
096
097  private final long scanTimeoutNs;
098
099  private final long pauseNs;
100
101  private final int maxAttempts;
102
103  private final int startLogErrorsCnt;
104
105  RawAsyncTableImpl(AsyncConnectionImpl conn, Timer retryTimer, AsyncTableBuilderBase<?> builder) {
106    this.conn = conn;
107    this.retryTimer = retryTimer;
108    this.tableName = builder.tableName;
109    this.rpcTimeoutNs = builder.rpcTimeoutNs;
110    this.readRpcTimeoutNs = builder.readRpcTimeoutNs;
111    this.writeRpcTimeoutNs = builder.writeRpcTimeoutNs;
112    this.operationTimeoutNs = builder.operationTimeoutNs;
113    this.scanTimeoutNs = builder.scanTimeoutNs;
114    this.pauseNs = builder.pauseNs;
115    this.maxAttempts = builder.maxAttempts;
116    this.startLogErrorsCnt = builder.startLogErrorsCnt;
117    this.defaultScannerCaching = tableName.isSystemTable() ? conn.connConf.getMetaScannerCaching()
118      : conn.connConf.getScannerCaching();
119    this.defaultScannerMaxResultSize = conn.connConf.getScannerMaxResultSize();
120  }
121
122  @Override
123  public TableName getName() {
124    return tableName;
125  }
126
127  @Override
128  public Configuration getConfiguration() {
129    return conn.getConfiguration();
130  }
131
132  @FunctionalInterface
133  private interface Converter<D, I, S> {
134    D convert(I info, S src) throws IOException;
135  }
136
137  @FunctionalInterface
138  private interface RpcCall<RESP, REQ> {
139    void call(ClientService.Interface stub, HBaseRpcController controller, REQ req,
140        RpcCallback<RESP> done);
141  }
142
143  private static <REQ, PREQ, PRESP, RESP> CompletableFuture<RESP> call(
144      HBaseRpcController controller, HRegionLocation loc, ClientService.Interface stub, REQ req,
145      Converter<PREQ, byte[], REQ> reqConvert, RpcCall<PRESP, PREQ> rpcCall,
146      Converter<RESP, HBaseRpcController, PRESP> respConverter) {
147    CompletableFuture<RESP> future = new CompletableFuture<>();
148    try {
149      rpcCall.call(stub, controller, reqConvert.convert(loc.getRegion().getRegionName(), req),
150        new RpcCallback<PRESP>() {
151
152          @Override
153          public void run(PRESP resp) {
154            if (controller.failed()) {
155              future.completeExceptionally(controller.getFailed());
156            } else {
157              try {
158                future.complete(respConverter.convert(controller, resp));
159              } catch (IOException e) {
160                future.completeExceptionally(e);
161              }
162            }
163          }
164        });
165    } catch (IOException e) {
166      future.completeExceptionally(e);
167    }
168    return future;
169  }
170
171  private static <REQ, RESP> CompletableFuture<RESP> mutate(HBaseRpcController controller,
172      HRegionLocation loc, ClientService.Interface stub, REQ req,
173      Converter<MutateRequest, byte[], REQ> reqConvert,
174      Converter<RESP, HBaseRpcController, MutateResponse> respConverter) {
175    return call(controller, loc, stub, req, reqConvert, (s, c, r, done) -> s.mutate(c, r, done),
176      respConverter);
177  }
178
179  private static <REQ> CompletableFuture<Void> voidMutate(HBaseRpcController controller,
180      HRegionLocation loc, ClientService.Interface stub, REQ req,
181      Converter<MutateRequest, byte[], REQ> reqConvert) {
182    return mutate(controller, loc, stub, req, reqConvert, (c, resp) -> {
183      return null;
184    });
185  }
186
187  private static Result toResult(HBaseRpcController controller, MutateResponse resp)
188      throws IOException {
189    if (!resp.hasResult()) {
190      return null;
191    }
192    return ProtobufUtil.toResult(resp.getResult(), controller.cellScanner());
193  }
194
195  @FunctionalInterface
196  private interface NoncedConverter<D, I, S> {
197    D convert(I info, S src, long nonceGroup, long nonce) throws IOException;
198  }
199
200  private <REQ, RESP> CompletableFuture<RESP> noncedMutate(long nonceGroup, long nonce,
201      HBaseRpcController controller, HRegionLocation loc, ClientService.Interface stub, REQ req,
202      NoncedConverter<MutateRequest, byte[], REQ> reqConvert,
203      Converter<RESP, HBaseRpcController, MutateResponse> respConverter) {
204    return mutate(controller, loc, stub, req,
205      (info, src) -> reqConvert.convert(info, src, nonceGroup, nonce), respConverter);
206  }
207
208  private <T> SingleRequestCallerBuilder<T> newCaller(byte[] row, long rpcTimeoutNs) {
209    return conn.callerFactory.<T> single().table(tableName).row(row)
210      .rpcTimeout(rpcTimeoutNs, TimeUnit.NANOSECONDS)
211      .operationTimeout(operationTimeoutNs, TimeUnit.NANOSECONDS)
212      .pause(pauseNs, TimeUnit.NANOSECONDS).maxAttempts(maxAttempts)
213      .startLogErrorsCnt(startLogErrorsCnt);
214  }
215
216  private <T> SingleRequestCallerBuilder<T> newCaller(Row row, long rpcTimeoutNs) {
217    return newCaller(row.getRow(), rpcTimeoutNs);
218  }
219
220  private CompletableFuture<Result> get(Get get, int replicaId) {
221    return this.<Result> newCaller(get, readRpcTimeoutNs)
222      .action((controller, loc, stub) -> RawAsyncTableImpl
223        .<Get, GetRequest, GetResponse, Result> call(controller, loc, stub, get,
224          RequestConverter::buildGetRequest, (s, c, req, done) -> s.get(c, req, done),
225          (c, resp) -> ProtobufUtil.toResult(resp.getResult(), c.cellScanner())))
226      .replicaId(replicaId).call();
227  }
228
229  @Override
230  public CompletableFuture<Result> get(Get get) {
231    return timelineConsistentRead(conn.getLocator(), tableName, get, get.getRow(),
232      RegionLocateType.CURRENT, replicaId -> get(get, replicaId), readRpcTimeoutNs,
233      conn.connConf.getPrimaryCallTimeoutNs(), retryTimer);
234  }
235
236  @Override
237  public CompletableFuture<Void> put(Put put) {
238    return this.<Void> newCaller(put, writeRpcTimeoutNs)
239      .action((controller, loc, stub) -> RawAsyncTableImpl.<Put> voidMutate(controller, loc, stub,
240        put, RequestConverter::buildMutateRequest))
241      .call();
242  }
243
244  @Override
245  public CompletableFuture<Void> delete(Delete delete) {
246    return this.<Void> newCaller(delete, writeRpcTimeoutNs)
247      .action((controller, loc, stub) -> RawAsyncTableImpl.<Delete> voidMutate(controller, loc,
248        stub, delete, RequestConverter::buildMutateRequest))
249      .call();
250  }
251
252  @Override
253  public CompletableFuture<Result> append(Append append) {
254    checkHasFamilies(append);
255    long nonceGroup = conn.getNonceGenerator().getNonceGroup();
256    long nonce = conn.getNonceGenerator().newNonce();
257    return this.<Result> newCaller(append, rpcTimeoutNs)
258      .action(
259        (controller, loc, stub) -> this.<Append, Result> noncedMutate(nonceGroup, nonce, controller,
260          loc, stub, append, RequestConverter::buildMutateRequest, RawAsyncTableImpl::toResult))
261      .call();
262  }
263
264  @Override
265  public CompletableFuture<Result> increment(Increment increment) {
266    checkHasFamilies(increment);
267    long nonceGroup = conn.getNonceGenerator().getNonceGroup();
268    long nonce = conn.getNonceGenerator().newNonce();
269    return this.<Result> newCaller(increment, rpcTimeoutNs)
270      .action((controller, loc, stub) -> this.<Increment, Result> noncedMutate(nonceGroup, nonce,
271        controller, loc, stub, increment, RequestConverter::buildMutateRequest,
272        RawAsyncTableImpl::toResult))
273      .call();
274  }
275
276  private final class CheckAndMutateBuilderImpl implements CheckAndMutateBuilder {
277
278    private final byte[] row;
279
280    private final byte[] family;
281
282    private byte[] qualifier;
283
284    private TimeRange timeRange;
285
286    private CompareOperator op;
287
288    private byte[] value;
289
290    public CheckAndMutateBuilderImpl(byte[] row, byte[] family) {
291      this.row = Preconditions.checkNotNull(row, "row is null");
292      this.family = Preconditions.checkNotNull(family, "family is null");
293    }
294
295    @Override
296    public CheckAndMutateBuilder qualifier(byte[] qualifier) {
297      this.qualifier = Preconditions.checkNotNull(qualifier, "qualifier is null. Consider using" +
298        " an empty byte array, or just do not call this method if you want a null qualifier");
299      return this;
300    }
301
302    @Override
303    public CheckAndMutateBuilder timeRange(TimeRange timeRange) {
304      this.timeRange = timeRange;
305      return this;
306    }
307
308    @Override
309    public CheckAndMutateBuilder ifNotExists() {
310      this.op = CompareOperator.EQUAL;
311      this.value = null;
312      return this;
313    }
314
315    @Override
316    public CheckAndMutateBuilder ifMatches(CompareOperator compareOp, byte[] value) {
317      this.op = Preconditions.checkNotNull(compareOp, "compareOp is null");
318      this.value = Preconditions.checkNotNull(value, "value is null");
319      return this;
320    }
321
322    private void preCheck() {
323      Preconditions.checkNotNull(op, "condition is null. You need to specify the condition by" +
324        " calling ifNotExists/ifEquals/ifMatches before executing the request");
325    }
326
327    @Override
328    public CompletableFuture<Boolean> thenPut(Put put) {
329      preCheck();
330      return RawAsyncTableImpl.this.<Boolean> newCaller(row, rpcTimeoutNs)
331        .action((controller, loc, stub) -> RawAsyncTableImpl.<Put, Boolean> mutate(controller, loc,
332          stub, put,
333          (rn, p) -> RequestConverter.buildMutateRequest(rn, row, family, qualifier,
334            new BinaryComparator(value), CompareType.valueOf(op.name()), timeRange, p),
335          (c, r) -> r.getProcessed()))
336        .call();
337    }
338
339    @Override
340    public CompletableFuture<Boolean> thenDelete(Delete delete) {
341      preCheck();
342      return RawAsyncTableImpl.this.<Boolean> newCaller(row, rpcTimeoutNs)
343        .action((controller, loc, stub) -> RawAsyncTableImpl.<Delete, Boolean> mutate(controller,
344          loc, stub, delete,
345          (rn, d) -> RequestConverter.buildMutateRequest(rn, row, family, qualifier,
346            new BinaryComparator(value), CompareType.valueOf(op.name()), timeRange, d),
347          (c, r) -> r.getProcessed()))
348        .call();
349    }
350
351    @Override
352    public CompletableFuture<Boolean> thenMutate(RowMutations mutation) {
353      preCheck();
354      return RawAsyncTableImpl.this.<Boolean> newCaller(mutation, rpcTimeoutNs)
355        .action((controller, loc, stub) -> RawAsyncTableImpl.<Boolean> mutateRow(controller, loc,
356          stub, mutation,
357          (rn, rm) -> RequestConverter.buildMutateRequest(rn, row, family, qualifier,
358            new BinaryComparator(value), CompareType.valueOf(op.name()), timeRange, rm),
359          resp -> resp.getExists()))
360        .call();
361    }
362  }
363
364  @Override
365  public CheckAndMutateBuilder checkAndMutate(byte[] row, byte[] family) {
366    return new CheckAndMutateBuilderImpl(row, family);
367  }
368
369  // We need the MultiRequest when constructing the org.apache.hadoop.hbase.client.MultiResponse,
370  // so here I write a new method as I do not want to change the abstraction of call method.
371  private static <RESP> CompletableFuture<RESP> mutateRow(HBaseRpcController controller,
372      HRegionLocation loc, ClientService.Interface stub, RowMutations mutation,
373      Converter<MultiRequest, byte[], RowMutations> reqConvert,
374      Function<Result, RESP> respConverter) {
375    CompletableFuture<RESP> future = new CompletableFuture<>();
376    try {
377      byte[] regionName = loc.getRegion().getRegionName();
378      MultiRequest req = reqConvert.convert(regionName, mutation);
379      stub.multi(controller, req, new RpcCallback<MultiResponse>() {
380
381        @Override
382        public void run(MultiResponse resp) {
383          if (controller.failed()) {
384            future.completeExceptionally(controller.getFailed());
385          } else {
386            try {
387              org.apache.hadoop.hbase.client.MultiResponse multiResp =
388                ResponseConverter.getResults(req, resp, controller.cellScanner());
389              Throwable ex = multiResp.getException(regionName);
390              if (ex != null) {
391                future.completeExceptionally(ex instanceof IOException ? ex
392                  : new IOException(
393                    "Failed to mutate row: " + Bytes.toStringBinary(mutation.getRow()), ex));
394              } else {
395                future.complete(respConverter
396                  .apply((Result) multiResp.getResults().get(regionName).result.get(0)));
397              }
398            } catch (IOException e) {
399              future.completeExceptionally(e);
400            }
401          }
402        }
403      });
404    } catch (IOException e) {
405      future.completeExceptionally(e);
406    }
407    return future;
408  }
409
410  @Override
411  public CompletableFuture<Void> mutateRow(RowMutations mutation) {
412    return this.<Void> newCaller(mutation, writeRpcTimeoutNs).action((controller, loc,
413        stub) -> RawAsyncTableImpl.<Void> mutateRow(controller, loc, stub, mutation, (rn, rm) -> {
414          RegionAction.Builder regionMutationBuilder = RequestConverter.buildRegionAction(rn, rm);
415          regionMutationBuilder.setAtomic(true);
416          return MultiRequest.newBuilder().addRegionAction(regionMutationBuilder.build()).build();
417        }, resp -> null))
418      .call();
419  }
420
421  private Scan setDefaultScanConfig(Scan scan) {
422    // always create a new scan object as we may reset the start row later.
423    Scan newScan = ReflectionUtils.newInstance(scan.getClass(), scan);
424    if (newScan.getCaching() <= 0) {
425      newScan.setCaching(defaultScannerCaching);
426    }
427    if (newScan.getMaxResultSize() <= 0) {
428      newScan.setMaxResultSize(defaultScannerMaxResultSize);
429    }
430    return newScan;
431  }
432
433  public void scan(Scan scan, AdvancedScanResultConsumer consumer) {
434    new AsyncClientScanner(setDefaultScanConfig(scan), consumer, tableName, conn, retryTimer,
435      pauseNs, maxAttempts, scanTimeoutNs, readRpcTimeoutNs, startLogErrorsCnt).start();
436  }
437
438  private long resultSize2CacheSize(long maxResultSize) {
439    // * 2 if possible
440    return maxResultSize > Long.MAX_VALUE / 2 ? maxResultSize : maxResultSize * 2;
441  }
442
443  @Override
444  public ResultScanner getScanner(Scan scan) {
445    return new AsyncTableResultScanner(this, ReflectionUtils.newInstance(scan.getClass(), scan),
446      resultSize2CacheSize(
447        scan.getMaxResultSize() > 0 ? scan.getMaxResultSize() : defaultScannerMaxResultSize));
448  }
449
450  @Override
451  public CompletableFuture<List<Result>> scanAll(Scan scan) {
452    CompletableFuture<List<Result>> future = new CompletableFuture<>();
453    List<Result> scanResults = new ArrayList<>();
454    scan(scan, new AdvancedScanResultConsumer() {
455
456      @Override
457      public void onNext(Result[] results, ScanController controller) {
458        scanResults.addAll(Arrays.asList(results));
459      }
460
461      @Override
462      public void onError(Throwable error) {
463        future.completeExceptionally(error);
464      }
465
466      @Override
467      public void onComplete() {
468        future.complete(scanResults);
469      }
470    });
471    return future;
472  }
473
474  @Override
475  public List<CompletableFuture<Result>> get(List<Get> gets) {
476    return batch(gets, readRpcTimeoutNs);
477  }
478
479  @Override
480  public List<CompletableFuture<Void>> put(List<Put> puts) {
481    return voidMutate(puts);
482  }
483
484  @Override
485  public List<CompletableFuture<Void>> delete(List<Delete> deletes) {
486    return voidMutate(deletes);
487  }
488
489  @Override
490  public <T> List<CompletableFuture<T>> batch(List<? extends Row> actions) {
491    return batch(actions, rpcTimeoutNs);
492  }
493
494  private List<CompletableFuture<Void>> voidMutate(List<? extends Row> actions) {
495    return this.<Object> batch(actions, writeRpcTimeoutNs).stream()
496      .map(f -> f.<Void> thenApply(r -> null)).collect(toList());
497  }
498
499  private <T> List<CompletableFuture<T>> batch(List<? extends Row> actions, long rpcTimeoutNs) {
500    return conn.callerFactory.batch().table(tableName).actions(actions)
501      .operationTimeout(operationTimeoutNs, TimeUnit.NANOSECONDS)
502      .rpcTimeout(rpcTimeoutNs, TimeUnit.NANOSECONDS).pause(pauseNs, TimeUnit.NANOSECONDS)
503      .maxAttempts(maxAttempts).startLogErrorsCnt(startLogErrorsCnt).call();
504  }
505
506  @Override
507  public long getRpcTimeout(TimeUnit unit) {
508    return unit.convert(rpcTimeoutNs, TimeUnit.NANOSECONDS);
509  }
510
511  @Override
512  public long getReadRpcTimeout(TimeUnit unit) {
513    return unit.convert(readRpcTimeoutNs, TimeUnit.NANOSECONDS);
514  }
515
516  @Override
517  public long getWriteRpcTimeout(TimeUnit unit) {
518    return unit.convert(writeRpcTimeoutNs, TimeUnit.NANOSECONDS);
519  }
520
521  @Override
522  public long getOperationTimeout(TimeUnit unit) {
523    return unit.convert(operationTimeoutNs, TimeUnit.NANOSECONDS);
524  }
525
526  @Override
527  public long getScanTimeout(TimeUnit unit) {
528    return unit.convert(scanTimeoutNs, TimeUnit.NANOSECONDS);
529  }
530
531  private <S, R> CompletableFuture<R> coprocessorService(Function<RpcChannel, S> stubMaker,
532      ServiceCaller<S, R> callable, RegionInfo region, byte[] row) {
533    RegionCoprocessorRpcChannelImpl channel = new RegionCoprocessorRpcChannelImpl(conn, tableName,
534      region, row, rpcTimeoutNs, operationTimeoutNs);
535    S stub = stubMaker.apply(channel);
536    CompletableFuture<R> future = new CompletableFuture<>();
537    ClientCoprocessorRpcController controller = new ClientCoprocessorRpcController();
538    callable.call(stub, controller, resp -> {
539      if (controller.failed()) {
540        future.completeExceptionally(controller.getFailed());
541      } else {
542        future.complete(resp);
543      }
544    });
545    return future;
546  }
547
548  @Override
549  public <S, R> CompletableFuture<R> coprocessorService(Function<RpcChannel, S> stubMaker,
550      ServiceCaller<S, R> callable, byte[] row) {
551    return coprocessorService(stubMaker, callable, null, row);
552  }
553
554  private boolean locateFinished(RegionInfo region, byte[] endKey, boolean endKeyInclusive) {
555    if (isEmptyStopRow(endKey)) {
556      if (isEmptyStopRow(region.getEndKey())) {
557        return true;
558      }
559      return false;
560    } else {
561      if (isEmptyStopRow(region.getEndKey())) {
562        return true;
563      }
564      int c = Bytes.compareTo(endKey, region.getEndKey());
565      // 1. if the region contains endKey
566      // 2. endKey is equal to the region's endKey and we do not want to include endKey.
567      return c < 0 || c == 0 && !endKeyInclusive;
568    }
569  }
570
571  private <S, R> void onLocateComplete(Function<RpcChannel, S> stubMaker,
572      ServiceCaller<S, R> callable, CoprocessorCallback<R> callback, List<HRegionLocation> locs,
573      byte[] endKey, boolean endKeyInclusive, AtomicBoolean locateFinished,
574      AtomicInteger unfinishedRequest, HRegionLocation loc, Throwable error) {
575    if (error != null) {
576      callback.onError(error);
577      return;
578    }
579    unfinishedRequest.incrementAndGet();
580    RegionInfo region = loc.getRegion();
581    if (locateFinished(region, endKey, endKeyInclusive)) {
582      locateFinished.set(true);
583    } else {
584      addListener(
585        conn.getLocator().getRegionLocation(tableName, region.getEndKey(), RegionLocateType.CURRENT,
586          operationTimeoutNs),
587        (l, e) -> onLocateComplete(stubMaker, callable, callback, locs, endKey, endKeyInclusive,
588          locateFinished, unfinishedRequest, l, e));
589    }
590    addListener(coprocessorService(stubMaker, callable, region, region.getStartKey()), (r, e) -> {
591      if (e != null) {
592        callback.onRegionError(region, e);
593      } else {
594        callback.onRegionComplete(region, r);
595      }
596      if (unfinishedRequest.decrementAndGet() == 0 && locateFinished.get()) {
597        callback.onComplete();
598      }
599    });
600  }
601
602  private final class CoprocessorServiceBuilderImpl<S, R>
603      implements CoprocessorServiceBuilder<S, R> {
604
605    private final Function<RpcChannel, S> stubMaker;
606
607    private final ServiceCaller<S, R> callable;
608
609    private final CoprocessorCallback<R> callback;
610
611    private byte[] startKey = HConstants.EMPTY_START_ROW;
612
613    private boolean startKeyInclusive;
614
615    private byte[] endKey = HConstants.EMPTY_END_ROW;
616
617    private boolean endKeyInclusive;
618
619    public CoprocessorServiceBuilderImpl(Function<RpcChannel, S> stubMaker,
620        ServiceCaller<S, R> callable, CoprocessorCallback<R> callback) {
621      this.stubMaker = Preconditions.checkNotNull(stubMaker, "stubMaker is null");
622      this.callable = Preconditions.checkNotNull(callable, "callable is null");
623      this.callback = Preconditions.checkNotNull(callback, "callback is null");
624    }
625
626    @Override
627    public CoprocessorServiceBuilderImpl<S, R> fromRow(byte[] startKey, boolean inclusive) {
628      this.startKey = Preconditions.checkNotNull(startKey,
629        "startKey is null. Consider using" +
630          " an empty byte array, or just do not call this method if you want to start selection" +
631          " from the first region");
632      this.startKeyInclusive = inclusive;
633      return this;
634    }
635
636    @Override
637    public CoprocessorServiceBuilderImpl<S, R> toRow(byte[] endKey, boolean inclusive) {
638      this.endKey = Preconditions.checkNotNull(endKey,
639        "endKey is null. Consider using" +
640          " an empty byte array, or just do not call this method if you want to continue" +
641          " selection to the last region");
642      this.endKeyInclusive = inclusive;
643      return this;
644    }
645
646    @Override
647    public void execute() {
648      addListener(conn.getLocator().getRegionLocation(tableName, startKey,
649        startKeyInclusive ? RegionLocateType.CURRENT : RegionLocateType.AFTER, operationTimeoutNs),
650        (loc, error) -> onLocateComplete(stubMaker, callable, callback, new ArrayList<>(), endKey,
651          endKeyInclusive, new AtomicBoolean(false), new AtomicInteger(0), loc, error));
652    }
653  }
654
655  @Override
656  public <S, R> CoprocessorServiceBuilder<S, R> coprocessorService(
657      Function<RpcChannel, S> stubMaker, ServiceCaller<S, R> callable,
658      CoprocessorCallback<R> callback) {
659    return new CoprocessorServiceBuilderImpl<>(stubMaker, callable, callback);
660  }
661}