001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import static org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction.SAFEMODE_GET;
021
022import edu.umd.cs.findbugs.annotations.CheckForNull;
023import java.io.ByteArrayInputStream;
024import java.io.DataInputStream;
025import java.io.EOFException;
026import java.io.FileNotFoundException;
027import java.io.IOException;
028import java.io.InterruptedIOException;
029import java.lang.reflect.InvocationTargetException;
030import java.lang.reflect.Method;
031import java.net.InetSocketAddress;
032import java.net.URI;
033import java.util.ArrayList;
034import java.util.Arrays;
035import java.util.Collection;
036import java.util.Collections;
037import java.util.HashMap;
038import java.util.HashSet;
039import java.util.Iterator;
040import java.util.List;
041import java.util.Locale;
042import java.util.Map;
043import java.util.Optional;
044import java.util.Set;
045import java.util.Vector;
046import java.util.concurrent.ConcurrentHashMap;
047import java.util.concurrent.ExecutionException;
048import java.util.concurrent.ExecutorService;
049import java.util.concurrent.Executors;
050import java.util.concurrent.Future;
051import java.util.concurrent.FutureTask;
052import java.util.concurrent.ThreadPoolExecutor;
053import java.util.concurrent.TimeUnit;
054import java.util.regex.Pattern;
055import org.apache.commons.lang3.ArrayUtils;
056import org.apache.hadoop.conf.Configuration;
057import org.apache.hadoop.fs.BlockLocation;
058import org.apache.hadoop.fs.FSDataInputStream;
059import org.apache.hadoop.fs.FSDataOutputStream;
060import org.apache.hadoop.fs.FileStatus;
061import org.apache.hadoop.fs.FileSystem;
062import org.apache.hadoop.fs.FileUtil;
063import org.apache.hadoop.fs.Path;
064import org.apache.hadoop.fs.PathFilter;
065import org.apache.hadoop.fs.StorageType;
066import org.apache.hadoop.fs.permission.FsPermission;
067import org.apache.hadoop.hbase.ClusterId;
068import org.apache.hadoop.hbase.HConstants;
069import org.apache.hadoop.hbase.HDFSBlocksDistribution;
070import org.apache.hadoop.hbase.TableName;
071import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
072import org.apache.hadoop.hbase.client.RegionInfo;
073import org.apache.hadoop.hbase.client.RegionInfoBuilder;
074import org.apache.hadoop.hbase.exceptions.DeserializationException;
075import org.apache.hadoop.hbase.fs.HFileSystem;
076import org.apache.hadoop.hbase.io.HFileLink;
077import org.apache.hadoop.hbase.master.HMaster;
078import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
079import org.apache.hadoop.hdfs.DFSClient;
080import org.apache.hadoop.hdfs.DFSHedgedReadMetrics;
081import org.apache.hadoop.hdfs.DFSUtil;
082import org.apache.hadoop.hdfs.DistributedFileSystem;
083import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
084import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
085import org.apache.hadoop.hdfs.protocol.LocatedBlock;
086import org.apache.hadoop.io.IOUtils;
087import org.apache.hadoop.ipc.RemoteException;
088import org.apache.hadoop.util.Progressable;
089import org.apache.yetus.audience.InterfaceAudience;
090import org.slf4j.Logger;
091import org.slf4j.LoggerFactory;
092
093import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
094import org.apache.hbase.thirdparty.com.google.common.collect.Iterators;
095import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
096import org.apache.hbase.thirdparty.com.google.common.primitives.Ints;
097import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
098
099import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
100import org.apache.hadoop.hbase.shaded.protobuf.generated.FSProtos;
101
102/**
103 * Utility methods for interacting with the underlying file system.
104 */
105@InterfaceAudience.Private
106public final class FSUtils {
107  private static final Logger LOG = LoggerFactory.getLogger(FSUtils.class);
108
109  private static final String THREAD_POOLSIZE = "hbase.client.localityCheck.threadPoolSize";
110  private static final int DEFAULT_THREAD_POOLSIZE = 2;
111
112  /** Set to true on Windows platforms */
113  // currently only used in testing. TODO refactor into a test class
114  public static final boolean WINDOWS = System.getProperty("os.name").startsWith("Windows");
115
116  private FSUtils() {
117  }
118
119  /** Returns True is <code>fs</code> is instance of DistributedFileSystem n */
120  public static boolean isDistributedFileSystem(final FileSystem fs) throws IOException {
121    FileSystem fileSystem = fs;
122    // If passed an instance of HFileSystem, it fails instanceof DistributedFileSystem.
123    // Check its backing fs for dfs-ness.
124    if (fs instanceof HFileSystem) {
125      fileSystem = ((HFileSystem) fs).getBackingFs();
126    }
127    return fileSystem instanceof DistributedFileSystem;
128  }
129
130  /**
131   * Compare path component of the Path URI; e.g. if hdfs://a/b/c and /a/b/c, it will compare the
132   * '/a/b/c' part. If you passed in 'hdfs://a/b/c and b/c, it would return true. Does not consider
133   * schema; i.e. if schemas different but path or subpath matches, the two will equate.
134   * @param pathToSearch Path we will be trying to match.
135   * @return True if <code>pathTail</code> is tail on the path of <code>pathToSearch</code>
136   */
137  public static boolean isMatchingTail(final Path pathToSearch, final Path pathTail) {
138    Path tailPath = pathTail;
139    String tailName;
140    Path toSearch = pathToSearch;
141    String toSearchName;
142    boolean result = false;
143
144    if (pathToSearch.depth() != pathTail.depth()) {
145      return false;
146    }
147
148    do {
149      tailName = tailPath.getName();
150      if (tailName == null || tailName.isEmpty()) {
151        result = true;
152        break;
153      }
154      toSearchName = toSearch.getName();
155      if (toSearchName == null || toSearchName.isEmpty()) {
156        break;
157      }
158      // Move up a parent on each path for next go around. Path doesn't let us go off the end.
159      tailPath = tailPath.getParent();
160      toSearch = toSearch.getParent();
161    } while (tailName.equals(toSearchName));
162    return result;
163  }
164
165  /**
166   * Delete the region directory if exists.
167   * @return True if deleted the region directory.
168   */
169  public static boolean deleteRegionDir(final Configuration conf, final RegionInfo hri)
170    throws IOException {
171    Path rootDir = CommonFSUtils.getRootDir(conf);
172    FileSystem fs = rootDir.getFileSystem(conf);
173    return CommonFSUtils.deleteDirectory(fs,
174      new Path(CommonFSUtils.getTableDir(rootDir, hri.getTable()), hri.getEncodedName()));
175  }
176
177  /**
178   * Create the specified file on the filesystem. By default, this will:
179   * <ol>
180   * <li>overwrite the file if it exists</li>
181   * <li>apply the umask in the configuration (if it is enabled)</li>
182   * <li>use the fs configured buffer size (or 4096 if not set)</li>
183   * <li>use the configured column family replication or default replication if
184   * {@link ColumnFamilyDescriptorBuilder#DEFAULT_DFS_REPLICATION}</li>
185   * <li>use the default block size</li>
186   * <li>not track progress</li>
187   * </ol>
188   * @param conf         configurations
189   * @param fs           {@link FileSystem} on which to write the file
190   * @param path         {@link Path} to the file to write
191   * @param perm         permissions
192   * @param favoredNodes favored data nodes
193   * @return output stream to the created file
194   * @throws IOException if the file cannot be created
195   */
196  public static FSDataOutputStream create(Configuration conf, FileSystem fs, Path path,
197    FsPermission perm, InetSocketAddress[] favoredNodes) throws IOException {
198    if (fs instanceof HFileSystem) {
199      FileSystem backingFs = ((HFileSystem) fs).getBackingFs();
200      if (backingFs instanceof DistributedFileSystem) {
201        // Try to use the favoredNodes version via reflection to allow backwards-
202        // compatibility.
203        short replication = Short.parseShort(conf.get(ColumnFamilyDescriptorBuilder.DFS_REPLICATION,
204          String.valueOf(ColumnFamilyDescriptorBuilder.DEFAULT_DFS_REPLICATION)));
205        try {
206          return (FSDataOutputStream) (DistributedFileSystem.class
207            .getDeclaredMethod("create", Path.class, FsPermission.class, boolean.class, int.class,
208              short.class, long.class, Progressable.class, InetSocketAddress[].class)
209            .invoke(backingFs, path, perm, true, CommonFSUtils.getDefaultBufferSize(backingFs),
210              replication > 0 ? replication : CommonFSUtils.getDefaultReplication(backingFs, path),
211              CommonFSUtils.getDefaultBlockSize(backingFs, path), null, favoredNodes));
212        } catch (InvocationTargetException ite) {
213          // Function was properly called, but threw it's own exception.
214          throw new IOException(ite.getCause());
215        } catch (NoSuchMethodException e) {
216          LOG.debug("DFS Client does not support most favored nodes create; using default create");
217          LOG.trace("Ignoring; use default create", e);
218        } catch (IllegalArgumentException | SecurityException | IllegalAccessException e) {
219          LOG.debug("Ignoring (most likely Reflection related exception) " + e);
220        }
221      }
222    }
223    return CommonFSUtils.create(fs, path, perm, true);
224  }
225
226  /**
227   * Checks to see if the specified file system is available
228   * @param fs filesystem
229   * @throws IOException e
230   */
231  public static void checkFileSystemAvailable(final FileSystem fs) throws IOException {
232    if (!(fs instanceof DistributedFileSystem)) {
233      return;
234    }
235    IOException exception = null;
236    DistributedFileSystem dfs = (DistributedFileSystem) fs;
237    try {
238      if (dfs.exists(new Path("/"))) {
239        return;
240      }
241    } catch (IOException e) {
242      exception = e instanceof RemoteException ? ((RemoteException) e).unwrapRemoteException() : e;
243    }
244    try {
245      fs.close();
246    } catch (Exception e) {
247      LOG.error("file system close failed: ", e);
248    }
249    throw new IOException("File system is not available", exception);
250  }
251
252  /**
253   * Inquire the Active NameNode's safe mode status.
254   * @param dfs A DistributedFileSystem object representing the underlying HDFS.
255   * @return whether we're in safe mode
256   */
257  private static boolean isInSafeMode(DistributedFileSystem dfs) throws IOException {
258    return dfs.setSafeMode(SAFEMODE_GET, true);
259  }
260
261  /**
262   * Check whether dfs is in safemode.
263   */
264  public static void checkDfsSafeMode(final Configuration conf) throws IOException {
265    boolean isInSafeMode = false;
266    FileSystem fs = FileSystem.get(conf);
267    if (fs instanceof DistributedFileSystem) {
268      DistributedFileSystem dfs = (DistributedFileSystem) fs;
269      isInSafeMode = isInSafeMode(dfs);
270    }
271    if (isInSafeMode) {
272      throw new IOException("File system is in safemode, it can't be written now");
273    }
274  }
275
276  /**
277   * Verifies current version of file system
278   * @param fs      filesystem object
279   * @param rootdir root hbase directory
280   * @return null if no version file exists, version string otherwise
281   * @throws IOException              if the version file fails to open
282   * @throws DeserializationException if the version data cannot be translated into a version
283   */
284  public static String getVersion(FileSystem fs, Path rootdir)
285    throws IOException, DeserializationException {
286    final Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
287    FileStatus[] status = null;
288    try {
289      // hadoop 2.0 throws FNFE if directory does not exist.
290      // hadoop 1.0 returns null if directory does not exist.
291      status = fs.listStatus(versionFile);
292    } catch (FileNotFoundException fnfe) {
293      return null;
294    }
295    if (ArrayUtils.getLength(status) == 0) {
296      return null;
297    }
298    String version = null;
299    byte[] content = new byte[(int) status[0].getLen()];
300    FSDataInputStream s = fs.open(versionFile);
301    try {
302      IOUtils.readFully(s, content, 0, content.length);
303      if (ProtobufUtil.isPBMagicPrefix(content)) {
304        version = parseVersionFrom(content);
305      } else {
306        // Presume it pre-pb format.
307        try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(content))) {
308          version = dis.readUTF();
309        }
310      }
311    } catch (EOFException eof) {
312      LOG.warn("Version file was empty, odd, will try to set it.");
313    } finally {
314      s.close();
315    }
316    return version;
317  }
318
319  /**
320   * Parse the content of the ${HBASE_ROOTDIR}/hbase.version file.
321   * @param bytes The byte content of the hbase.version file
322   * @return The version found in the file as a String
323   * @throws DeserializationException if the version data cannot be translated into a version
324   */
325  static String parseVersionFrom(final byte[] bytes) throws DeserializationException {
326    ProtobufUtil.expectPBMagicPrefix(bytes);
327    int pblen = ProtobufUtil.lengthOfPBMagic();
328    FSProtos.HBaseVersionFileContent.Builder builder =
329      FSProtos.HBaseVersionFileContent.newBuilder();
330    try {
331      ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
332      return builder.getVersion();
333    } catch (IOException e) {
334      // Convert
335      throw new DeserializationException(e);
336    }
337  }
338
339  /**
340   * Create the content to write into the ${HBASE_ROOTDIR}/hbase.version file.
341   * @param version Version to persist
342   * @return Serialized protobuf with <code>version</code> content and a bit of pb magic for a
343   *         prefix.
344   */
345  static byte[] toVersionByteArray(final String version) {
346    FSProtos.HBaseVersionFileContent.Builder builder =
347      FSProtos.HBaseVersionFileContent.newBuilder();
348    return ProtobufUtil.prependPBMagic(builder.setVersion(version).build().toByteArray());
349  }
350
351  /**
352   * Verifies current version of file system
353   * @param fs      file system
354   * @param rootdir root directory of HBase installation
355   * @param message if true, issues a message on System.out
356   * @throws IOException              if the version file cannot be opened
357   * @throws DeserializationException if the contents of the version file cannot be parsed
358   */
359  public static void checkVersion(FileSystem fs, Path rootdir, boolean message)
360    throws IOException, DeserializationException {
361    checkVersion(fs, rootdir, message, 0, HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
362  }
363
364  /**
365   * Verifies current version of file system
366   * @param fs      file system
367   * @param rootdir root directory of HBase installation
368   * @param message if true, issues a message on System.out
369   * @param wait    wait interval
370   * @param retries number of times to retry
371   * @throws IOException              if the version file cannot be opened
372   * @throws DeserializationException if the contents of the version file cannot be parsed
373   */
374  public static void checkVersion(FileSystem fs, Path rootdir, boolean message, int wait,
375    int retries) throws IOException, DeserializationException {
376    String version = getVersion(fs, rootdir);
377    String msg;
378    if (version == null) {
379      if (!metaRegionExists(fs, rootdir)) {
380        // rootDir is empty (no version file and no root region)
381        // just create new version file (HBASE-1195)
382        setVersion(fs, rootdir, wait, retries);
383        return;
384      } else {
385        msg = "hbase.version file is missing. Is your hbase.rootdir valid? "
386          + "You can restore hbase.version file by running 'HBCK2 filesystem -fix'. "
387          + "See https://github.com/apache/hbase-operator-tools/tree/master/hbase-hbck2";
388      }
389    } else if (version.compareTo(HConstants.FILE_SYSTEM_VERSION) == 0) {
390      return;
391    } else {
392      msg = "HBase file layout needs to be upgraded. Current filesystem version is " + version
393        + " but software requires version " + HConstants.FILE_SYSTEM_VERSION
394        + ". Consult http://hbase.apache.org/book.html for further information about "
395        + "upgrading HBase.";
396    }
397
398    // version is deprecated require migration
399    // Output on stdout so user sees it in terminal.
400    if (message) {
401      System.out.println("WARNING! " + msg);
402    }
403    throw new FileSystemVersionException(msg);
404  }
405
406  /**
407   * Sets version of file system
408   * @param fs      filesystem object
409   * @param rootdir hbase root
410   * @throws IOException e
411   */
412  public static void setVersion(FileSystem fs, Path rootdir) throws IOException {
413    setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, 0,
414      HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
415  }
416
417  /**
418   * Sets version of file system
419   * @param fs      filesystem object
420   * @param rootdir hbase root
421   * @param wait    time to wait for retry
422   * @param retries number of times to retry before failing
423   * @throws IOException e
424   */
425  public static void setVersion(FileSystem fs, Path rootdir, int wait, int retries)
426    throws IOException {
427    setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, wait, retries);
428  }
429
430  /**
431   * Sets version of file system
432   * @param fs      filesystem object
433   * @param rootdir hbase root directory
434   * @param version version to set
435   * @param wait    time to wait for retry
436   * @param retries number of times to retry before throwing an IOException
437   * @throws IOException e
438   */
439  public static void setVersion(FileSystem fs, Path rootdir, String version, int wait, int retries)
440    throws IOException {
441    Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
442    Path tempVersionFile = new Path(rootdir,
443      HConstants.HBASE_TEMP_DIRECTORY + Path.SEPARATOR + HConstants.VERSION_FILE_NAME);
444    while (true) {
445      try {
446        // Write the version to a temporary file
447        FSDataOutputStream s = fs.create(tempVersionFile);
448        try {
449          s.write(toVersionByteArray(version));
450          s.close();
451          s = null;
452          // Move the temp version file to its normal location. Returns false
453          // if the rename failed. Throw an IOE in that case.
454          if (!fs.rename(tempVersionFile, versionFile)) {
455            throw new IOException("Unable to move temp version file to " + versionFile);
456          }
457        } finally {
458          // Cleaning up the temporary if the rename failed would be trying
459          // too hard. We'll unconditionally create it again the next time
460          // through anyway, files are overwritten by default by create().
461
462          // Attempt to close the stream on the way out if it is still open.
463          try {
464            if (s != null) s.close();
465          } catch (IOException ignore) {
466          }
467        }
468        LOG.info("Created version file at " + rootdir.toString() + " with version=" + version);
469        return;
470      } catch (IOException e) {
471        if (retries > 0) {
472          LOG.debug("Unable to create version file at " + rootdir.toString() + ", retrying", e);
473          fs.delete(versionFile, false);
474          try {
475            if (wait > 0) {
476              Thread.sleep(wait);
477            }
478          } catch (InterruptedException ie) {
479            throw (InterruptedIOException) new InterruptedIOException().initCause(ie);
480          }
481          retries--;
482        } else {
483          throw e;
484        }
485      }
486    }
487  }
488
489  /**
490   * Checks that a cluster ID file exists in the HBase root directory
491   * @param fs      the root directory FileSystem
492   * @param rootdir the HBase root directory in HDFS
493   * @param wait    how long to wait between retries
494   * @return <code>true</code> if the file exists, otherwise <code>false</code>
495   * @throws IOException if checking the FileSystem fails
496   */
497  public static boolean checkClusterIdExists(FileSystem fs, Path rootdir, long wait)
498    throws IOException {
499    while (true) {
500      try {
501        Path filePath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
502        return fs.exists(filePath);
503      } catch (IOException ioe) {
504        if (wait > 0L) {
505          LOG.warn("Unable to check cluster ID file in {}, retrying in {}ms", rootdir, wait, ioe);
506          try {
507            Thread.sleep(wait);
508          } catch (InterruptedException e) {
509            Thread.currentThread().interrupt();
510            throw (InterruptedIOException) new InterruptedIOException().initCause(e);
511          }
512        } else {
513          throw ioe;
514        }
515      }
516    }
517  }
518
519  /**
520   * Returns the value of the unique cluster ID stored for this HBase instance.
521   * @param fs      the root directory FileSystem
522   * @param rootdir the path to the HBase root directory
523   * @return the unique cluster identifier
524   * @throws IOException if reading the cluster ID file fails
525   */
526  public static ClusterId getClusterId(FileSystem fs, Path rootdir) throws IOException {
527    Path idPath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
528    ClusterId clusterId = null;
529    FileStatus status = fs.exists(idPath) ? fs.getFileStatus(idPath) : null;
530    if (status != null) {
531      int len = Ints.checkedCast(status.getLen());
532      byte[] content = new byte[len];
533      FSDataInputStream in = fs.open(idPath);
534      try {
535        in.readFully(content);
536      } catch (EOFException eof) {
537        LOG.warn("Cluster ID file {} is empty", idPath);
538      } finally {
539        in.close();
540      }
541      try {
542        clusterId = ClusterId.parseFrom(content);
543      } catch (DeserializationException e) {
544        throw new IOException("content=" + Bytes.toString(content), e);
545      }
546      // If not pb'd, make it so.
547      if (!ProtobufUtil.isPBMagicPrefix(content)) {
548        String cid = null;
549        in = fs.open(idPath);
550        try {
551          cid = in.readUTF();
552          clusterId = new ClusterId(cid);
553        } catch (EOFException eof) {
554          LOG.warn("Cluster ID file {} is empty", idPath);
555        } finally {
556          in.close();
557        }
558        rewriteAsPb(fs, rootdir, idPath, clusterId);
559      }
560      return clusterId;
561    } else {
562      LOG.warn("Cluster ID file does not exist at {}", idPath);
563    }
564    return clusterId;
565  }
566
567  /**
568   *   */
569  private static void rewriteAsPb(final FileSystem fs, final Path rootdir, final Path p,
570    final ClusterId cid) throws IOException {
571    // Rewrite the file as pb. Move aside the old one first, write new
572    // then delete the moved-aside file.
573    Path movedAsideName = new Path(p + "." + EnvironmentEdgeManager.currentTime());
574    if (!fs.rename(p, movedAsideName)) throw new IOException("Failed rename of " + p);
575    setClusterId(fs, rootdir, cid, 100);
576    if (!fs.delete(movedAsideName, false)) {
577      throw new IOException("Failed delete of " + movedAsideName);
578    }
579    LOG.debug("Rewrote the hbase.id file as pb");
580  }
581
582  /**
583   * Writes a new unique identifier for this cluster to the "hbase.id" file in the HBase root
584   * directory. If any operations on the ID file fails, and {@code wait} is a positive value, the
585   * method will retry to produce the ID file until the thread is forcibly interrupted.
586   * @param fs        the root directory FileSystem
587   * @param rootdir   the path to the HBase root directory
588   * @param clusterId the unique identifier to store
589   * @param wait      how long (in milliseconds) to wait between retries
590   * @throws IOException if writing to the FileSystem fails and no wait value
591   */
592  public static void setClusterId(final FileSystem fs, final Path rootdir,
593    final ClusterId clusterId, final long wait) throws IOException {
594
595    final Path idFile = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
596    final Path tempDir = new Path(rootdir, HConstants.HBASE_TEMP_DIRECTORY);
597    final Path tempIdFile = new Path(tempDir, HConstants.CLUSTER_ID_FILE_NAME);
598
599    LOG.debug("Create cluster ID file [{}] with ID: {}", idFile, clusterId);
600
601    while (true) {
602      Optional<IOException> failure = Optional.empty();
603
604      LOG.debug("Write the cluster ID file to a temporary location: {}", tempIdFile);
605      try (FSDataOutputStream s = fs.create(tempIdFile)) {
606        s.write(clusterId.toByteArray());
607      } catch (IOException ioe) {
608        failure = Optional.of(ioe);
609      }
610
611      if (!failure.isPresent()) {
612        try {
613          LOG.debug("Move the temporary cluster ID file to its target location [{}]:[{}]",
614            tempIdFile, idFile);
615
616          if (!fs.rename(tempIdFile, idFile)) {
617            failure =
618              Optional.of(new IOException("Unable to move temp cluster ID file to " + idFile));
619          }
620        } catch (IOException ioe) {
621          failure = Optional.of(ioe);
622        }
623      }
624
625      if (failure.isPresent()) {
626        final IOException cause = failure.get();
627        if (wait > 0L) {
628          LOG.warn("Unable to create cluster ID file in {}, retrying in {}ms", rootdir, wait,
629            cause);
630          try {
631            Thread.sleep(wait);
632          } catch (InterruptedException e) {
633            Thread.currentThread().interrupt();
634            throw (InterruptedIOException) new InterruptedIOException().initCause(e);
635          }
636          continue;
637        } else {
638          throw cause;
639        }
640      } else {
641        return;
642      }
643    }
644  }
645
646  /**
647   * If DFS, check safe mode and if so, wait until we clear it.
648   * @param conf configuration
649   * @param wait Sleep between retries
650   * @throws IOException e
651   */
652  public static void waitOnSafeMode(final Configuration conf, final long wait) throws IOException {
653    FileSystem fs = FileSystem.get(conf);
654    if (!(fs instanceof DistributedFileSystem)) return;
655    DistributedFileSystem dfs = (DistributedFileSystem) fs;
656    // Make sure dfs is not in safe mode
657    while (isInSafeMode(dfs)) {
658      LOG.info("Waiting for dfs to exit safe mode...");
659      try {
660        Thread.sleep(wait);
661      } catch (InterruptedException e) {
662        Thread.currentThread().interrupt();
663        throw (InterruptedIOException) new InterruptedIOException().initCause(e);
664      }
665    }
666  }
667
668  /**
669   * Checks if meta region exists
670   * @param fs      file system
671   * @param rootDir root directory of HBase installation
672   * @return true if exists
673   */
674  public static boolean metaRegionExists(FileSystem fs, Path rootDir) throws IOException {
675    Path metaRegionDir = getRegionDirFromRootDir(rootDir, RegionInfoBuilder.FIRST_META_REGIONINFO);
676    return fs.exists(metaRegionDir);
677  }
678
679  /**
680   * Compute HDFS block distribution of a given HdfsDataInputStream. All HdfsDataInputStreams are
681   * backed by a series of LocatedBlocks, which are fetched periodically from the namenode. This
682   * method retrieves those blocks from the input stream and uses them to calculate
683   * HDFSBlockDistribution. The underlying method in DFSInputStream does attempt to use locally
684   * cached blocks, but may hit the namenode if the cache is determined to be incomplete. The method
685   * also involves making copies of all LocatedBlocks rather than return the underlying blocks
686   * themselves.
687   */
688  static public HDFSBlocksDistribution
689    computeHDFSBlocksDistribution(HdfsDataInputStream inputStream) throws IOException {
690    List<LocatedBlock> blocks = inputStream.getAllBlocks();
691    HDFSBlocksDistribution blocksDistribution = new HDFSBlocksDistribution();
692    for (LocatedBlock block : blocks) {
693      String[] hosts = getHostsForLocations(block);
694      long len = block.getBlockSize();
695      StorageType[] storageTypes = block.getStorageTypes();
696      blocksDistribution.addHostsAndBlockWeight(hosts, len, storageTypes);
697    }
698    return blocksDistribution;
699  }
700
701  private static String[] getHostsForLocations(LocatedBlock block) {
702    DatanodeInfo[] locations = block.getLocations();
703    String[] hosts = new String[locations.length];
704    for (int i = 0; i < hosts.length; i++) {
705      hosts[i] = locations[i].getHostName();
706    }
707    return hosts;
708  }
709
710  /**
711   * Compute HDFS blocks distribution of a given file, or a portion of the file
712   * @param fs     file system
713   * @param status file status of the file
714   * @param start  start position of the portion
715   * @param length length of the portion
716   * @return The HDFS blocks distribution
717   */
718  static public HDFSBlocksDistribution computeHDFSBlocksDistribution(final FileSystem fs,
719    FileStatus status, long start, long length) throws IOException {
720    HDFSBlocksDistribution blocksDistribution = new HDFSBlocksDistribution();
721    BlockLocation[] blockLocations = fs.getFileBlockLocations(status, start, length);
722    addToHDFSBlocksDistribution(blocksDistribution, blockLocations);
723    return blocksDistribution;
724  }
725
726  /**
727   * Update blocksDistribution with blockLocations
728   * @param blocksDistribution the hdfs blocks distribution
729   * @param blockLocations     an array containing block location
730   */
731  static public void addToHDFSBlocksDistribution(HDFSBlocksDistribution blocksDistribution,
732    BlockLocation[] blockLocations) throws IOException {
733    for (BlockLocation bl : blockLocations) {
734      String[] hosts = bl.getHosts();
735      long len = bl.getLength();
736      StorageType[] storageTypes = bl.getStorageTypes();
737      blocksDistribution.addHostsAndBlockWeight(hosts, len, storageTypes);
738    }
739  }
740
741  // TODO move this method OUT of FSUtils. No dependencies to HMaster
742  /**
743   * Returns the total overall fragmentation percentage. Includes hbase:meta and -ROOT- as well.
744   * @param master The master defining the HBase root and file system
745   * @return A map for each table and its percentage (never null)
746   * @throws IOException When scanning the directory fails
747   */
748  public static int getTotalTableFragmentation(final HMaster master) throws IOException {
749    Map<String, Integer> map = getTableFragmentation(master);
750    return map.isEmpty() ? -1 : map.get("-TOTAL-");
751  }
752
753  /**
754   * Runs through the HBase rootdir and checks how many stores for each table have more than one
755   * file in them. Checks -ROOT- and hbase:meta too. The total percentage across all tables is
756   * stored under the special key "-TOTAL-".
757   * @param master The master defining the HBase root and file system.
758   * @return A map for each table and its percentage (never null).
759   * @throws IOException When scanning the directory fails.
760   */
761  public static Map<String, Integer> getTableFragmentation(final HMaster master)
762    throws IOException {
763    Path path = CommonFSUtils.getRootDir(master.getConfiguration());
764    // since HMaster.getFileSystem() is package private
765    FileSystem fs = path.getFileSystem(master.getConfiguration());
766    return getTableFragmentation(fs, path);
767  }
768
769  /**
770   * Runs through the HBase rootdir and checks how many stores for each table have more than one
771   * file in them. Checks -ROOT- and hbase:meta too. The total percentage across all tables is
772   * stored under the special key "-TOTAL-".
773   * @param fs           The file system to use
774   * @param hbaseRootDir The root directory to scan
775   * @return A map for each table and its percentage (never null)
776   * @throws IOException When scanning the directory fails
777   */
778  public static Map<String, Integer> getTableFragmentation(final FileSystem fs,
779    final Path hbaseRootDir) throws IOException {
780    Map<String, Integer> frags = new HashMap<>();
781    int cfCountTotal = 0;
782    int cfFragTotal = 0;
783    PathFilter regionFilter = new RegionDirFilter(fs);
784    PathFilter familyFilter = new FamilyDirFilter(fs);
785    List<Path> tableDirs = getTableDirs(fs, hbaseRootDir);
786    for (Path d : tableDirs) {
787      int cfCount = 0;
788      int cfFrag = 0;
789      FileStatus[] regionDirs = fs.listStatus(d, regionFilter);
790      for (FileStatus regionDir : regionDirs) {
791        Path dd = regionDir.getPath();
792        // else its a region name, now look in region for families
793        FileStatus[] familyDirs = fs.listStatus(dd, familyFilter);
794        for (FileStatus familyDir : familyDirs) {
795          cfCount++;
796          cfCountTotal++;
797          Path family = familyDir.getPath();
798          // now in family make sure only one file
799          FileStatus[] familyStatus = fs.listStatus(family);
800          if (familyStatus.length > 1) {
801            cfFrag++;
802            cfFragTotal++;
803          }
804        }
805      }
806      // compute percentage per table and store in result list
807      frags.put(CommonFSUtils.getTableName(d).getNameAsString(),
808        cfCount == 0 ? 0 : Math.round((float) cfFrag / cfCount * 100));
809    }
810    // set overall percentage for all tables
811    frags.put("-TOTAL-",
812      cfCountTotal == 0 ? 0 : Math.round((float) cfFragTotal / cfCountTotal * 100));
813    return frags;
814  }
815
816  public static void renameFile(FileSystem fs, Path src, Path dst) throws IOException {
817    if (fs.exists(dst) && !fs.delete(dst, false)) {
818      throw new IOException("Can not delete " + dst);
819    }
820    if (!fs.rename(src, dst)) {
821      throw new IOException("Can not rename from " + src + " to " + dst);
822    }
823  }
824
825  /**
826   * A {@link PathFilter} that returns only regular files.
827   */
828  static class FileFilter extends AbstractFileStatusFilter {
829    private final FileSystem fs;
830
831    public FileFilter(final FileSystem fs) {
832      this.fs = fs;
833    }
834
835    @Override
836    protected boolean accept(Path p, @CheckForNull Boolean isDir) {
837      try {
838        return isFile(fs, isDir, p);
839      } catch (IOException e) {
840        LOG.warn("Unable to verify if path={} is a regular file", p, e);
841        return false;
842      }
843    }
844  }
845
846  /**
847   * Directory filter that doesn't include any of the directories in the specified blacklist
848   */
849  public static class BlackListDirFilter extends AbstractFileStatusFilter {
850    private final FileSystem fs;
851    private List<String> blacklist;
852
853    /**
854     * Create a filter on the givem filesystem with the specified blacklist
855     * @param fs                     filesystem to filter
856     * @param directoryNameBlackList list of the names of the directories to filter. If
857     *                               <tt>null</tt>, all directories are returned
858     */
859    @SuppressWarnings("unchecked")
860    public BlackListDirFilter(final FileSystem fs, final List<String> directoryNameBlackList) {
861      this.fs = fs;
862      blacklist = (List<String>) (directoryNameBlackList == null
863        ? Collections.emptyList()
864        : directoryNameBlackList);
865    }
866
867    @Override
868    protected boolean accept(Path p, @CheckForNull Boolean isDir) {
869      if (!isValidName(p.getName())) {
870        return false;
871      }
872
873      try {
874        return isDirectory(fs, isDir, p);
875      } catch (IOException e) {
876        LOG.warn("An error occurred while verifying if [{}] is a valid directory."
877          + " Returning 'not valid' and continuing.", p, e);
878        return false;
879      }
880    }
881
882    protected boolean isValidName(final String name) {
883      return !blacklist.contains(name);
884    }
885  }
886
887  /**
888   * A {@link PathFilter} that only allows directories.
889   */
890  public static class DirFilter extends BlackListDirFilter {
891
892    public DirFilter(FileSystem fs) {
893      super(fs, null);
894    }
895  }
896
897  /**
898   * A {@link PathFilter} that returns usertable directories. To get all directories use the
899   * {@link BlackListDirFilter} with a <tt>null</tt> blacklist
900   */
901  public static class UserTableDirFilter extends BlackListDirFilter {
902    public UserTableDirFilter(FileSystem fs) {
903      super(fs, HConstants.HBASE_NON_TABLE_DIRS);
904    }
905
906    @Override
907    protected boolean isValidName(final String name) {
908      if (!super.isValidName(name)) return false;
909
910      try {
911        TableName.isLegalTableQualifierName(Bytes.toBytes(name));
912      } catch (IllegalArgumentException e) {
913        LOG.info("Invalid table name: {}", name);
914        return false;
915      }
916      return true;
917    }
918  }
919
920  public static List<Path> getTableDirs(final FileSystem fs, final Path rootdir)
921    throws IOException {
922    List<Path> tableDirs = new ArrayList<>();
923    Path baseNamespaceDir = new Path(rootdir, HConstants.BASE_NAMESPACE_DIR);
924    if (fs.exists(baseNamespaceDir)) {
925      for (FileStatus status : fs.globStatus(new Path(baseNamespaceDir, "*"))) {
926        tableDirs.addAll(FSUtils.getLocalTableDirs(fs, status.getPath()));
927      }
928    }
929    return tableDirs;
930  }
931
932  /**
933   * @return All the table directories under <code>rootdir</code>. Ignore non table hbase folders
934   *         such as .logs, .oldlogs, .corrupt folders.
935   */
936  public static List<Path> getLocalTableDirs(final FileSystem fs, final Path rootdir)
937    throws IOException {
938    // presumes any directory under hbase.rootdir is a table
939    FileStatus[] dirs = fs.listStatus(rootdir, new UserTableDirFilter(fs));
940    List<Path> tabledirs = new ArrayList<>(dirs.length);
941    for (FileStatus dir : dirs) {
942      tabledirs.add(dir.getPath());
943    }
944    return tabledirs;
945  }
946
947  /**
948   * Filter for all dirs that don't start with '.'
949   */
950  public static class RegionDirFilter extends AbstractFileStatusFilter {
951    // This pattern will accept 0.90+ style hex region dirs and older numeric region dir names.
952    final public static Pattern regionDirPattern = Pattern.compile("^[0-9a-f]*$");
953    final FileSystem fs;
954
955    public RegionDirFilter(FileSystem fs) {
956      this.fs = fs;
957    }
958
959    @Override
960    protected boolean accept(Path p, @CheckForNull Boolean isDir) {
961      if (!regionDirPattern.matcher(p.getName()).matches()) {
962        return false;
963      }
964
965      try {
966        return isDirectory(fs, isDir, p);
967      } catch (IOException ioe) {
968        // Maybe the file was moved or the fs was disconnected.
969        LOG.warn("Skipping file {} due to IOException", p, ioe);
970        return false;
971      }
972    }
973  }
974
975  /**
976   * Given a particular table dir, return all the regiondirs inside it, excluding files such as
977   * .tableinfo
978   * @param fs       A file system for the Path
979   * @param tableDir Path to a specific table directory &lt;hbase.rootdir&gt;/&lt;tabledir&gt;
980   * @return List of paths to valid region directories in table dir.
981   */
982  public static List<Path> getRegionDirs(final FileSystem fs, final Path tableDir)
983    throws IOException {
984    // assumes we are in a table dir.
985    List<FileStatus> rds = listStatusWithStatusFilter(fs, tableDir, new RegionDirFilter(fs));
986    if (rds == null) {
987      return Collections.emptyList();
988    }
989    List<Path> regionDirs = new ArrayList<>(rds.size());
990    for (FileStatus rdfs : rds) {
991      Path rdPath = rdfs.getPath();
992      regionDirs.add(rdPath);
993    }
994    return regionDirs;
995  }
996
997  public static Path getRegionDirFromRootDir(Path rootDir, RegionInfo region) {
998    return getRegionDirFromTableDir(CommonFSUtils.getTableDir(rootDir, region.getTable()), region);
999  }
1000
1001  public static Path getRegionDirFromTableDir(Path tableDir, RegionInfo region) {
1002    return getRegionDirFromTableDir(tableDir,
1003      ServerRegionReplicaUtil.getRegionInfoForFs(region).getEncodedName());
1004  }
1005
1006  public static Path getRegionDirFromTableDir(Path tableDir, String encodedRegionName) {
1007    return new Path(tableDir, encodedRegionName);
1008  }
1009
1010  /**
1011   * Filter for all dirs that are legal column family names. This is generally used for colfam dirs
1012   * &lt;hbase.rootdir&gt;/&lt;tabledir&gt;/&lt;regiondir&gt;/&lt;colfamdir&gt;.
1013   */
1014  public static class FamilyDirFilter extends AbstractFileStatusFilter {
1015    final FileSystem fs;
1016
1017    public FamilyDirFilter(FileSystem fs) {
1018      this.fs = fs;
1019    }
1020
1021    @Override
1022    protected boolean accept(Path p, @CheckForNull Boolean isDir) {
1023      try {
1024        // throws IAE if invalid
1025        ColumnFamilyDescriptorBuilder.isLegalColumnFamilyName(Bytes.toBytes(p.getName()));
1026      } catch (IllegalArgumentException iae) {
1027        // path name is an invalid family name and thus is excluded.
1028        return false;
1029      }
1030
1031      try {
1032        return isDirectory(fs, isDir, p);
1033      } catch (IOException ioe) {
1034        // Maybe the file was moved or the fs was disconnected.
1035        LOG.warn("Skipping file {} due to IOException", p, ioe);
1036        return false;
1037      }
1038    }
1039  }
1040
1041  /**
1042   * Given a particular region dir, return all the familydirs inside it
1043   * @param fs        A file system for the Path
1044   * @param regionDir Path to a specific region directory
1045   * @return List of paths to valid family directories in region dir.
1046   */
1047  public static List<Path> getFamilyDirs(final FileSystem fs, final Path regionDir)
1048    throws IOException {
1049    // assumes we are in a region dir.
1050    return getFilePaths(fs, regionDir, new FamilyDirFilter(fs));
1051  }
1052
1053  public static List<Path> getReferenceFilePaths(final FileSystem fs, final Path familyDir)
1054    throws IOException {
1055    return getFilePaths(fs, familyDir, new ReferenceFileFilter(fs));
1056  }
1057
1058  public static List<Path> getReferenceAndLinkFilePaths(final FileSystem fs, final Path familyDir)
1059    throws IOException {
1060    return getFilePaths(fs, familyDir, new ReferenceAndLinkFileFilter(fs));
1061  }
1062
1063  private static List<Path> getFilePaths(final FileSystem fs, final Path dir,
1064    final PathFilter pathFilter) throws IOException {
1065    FileStatus[] fds = fs.listStatus(dir, pathFilter);
1066    List<Path> files = new ArrayList<>(fds.length);
1067    for (FileStatus fdfs : fds) {
1068      Path fdPath = fdfs.getPath();
1069      files.add(fdPath);
1070    }
1071    return files;
1072  }
1073
1074  public static int getRegionReferenceAndLinkFileCount(final FileSystem fs, final Path p) {
1075    int result = 0;
1076    try {
1077      for (Path familyDir : getFamilyDirs(fs, p)) {
1078        result += getReferenceAndLinkFilePaths(fs, familyDir).size();
1079      }
1080    } catch (IOException e) {
1081      LOG.warn("Error Counting reference files.", e);
1082    }
1083    return result;
1084  }
1085
1086  public static class ReferenceAndLinkFileFilter implements PathFilter {
1087
1088    private final FileSystem fs;
1089
1090    public ReferenceAndLinkFileFilter(FileSystem fs) {
1091      this.fs = fs;
1092    }
1093
1094    @Override
1095    public boolean accept(Path rd) {
1096      try {
1097        // only files can be references.
1098        return !fs.getFileStatus(rd).isDirectory()
1099          && (StoreFileInfo.isReference(rd) || HFileLink.isHFileLink(rd));
1100      } catch (IOException ioe) {
1101        // Maybe the file was moved or the fs was disconnected.
1102        LOG.warn("Skipping file " + rd + " due to IOException", ioe);
1103        return false;
1104      }
1105    }
1106  }
1107
1108  /**
1109   * Filter for HFiles that excludes reference files.
1110   */
1111  public static class HFileFilter extends AbstractFileStatusFilter {
1112    final FileSystem fs;
1113
1114    public HFileFilter(FileSystem fs) {
1115      this.fs = fs;
1116    }
1117
1118    @Override
1119    protected boolean accept(Path p, @CheckForNull Boolean isDir) {
1120      if (!StoreFileInfo.isHFile(p) && !StoreFileInfo.isMobFile(p)) {
1121        return false;
1122      }
1123
1124      try {
1125        return isFile(fs, isDir, p);
1126      } catch (IOException ioe) {
1127        // Maybe the file was moved or the fs was disconnected.
1128        LOG.warn("Skipping file {} due to IOException", p, ioe);
1129        return false;
1130      }
1131    }
1132  }
1133
1134  /**
1135   * Filter for HFileLinks (StoreFiles and HFiles not included). the filter itself does not consider
1136   * if a link is file or not.
1137   */
1138  public static class HFileLinkFilter implements PathFilter {
1139
1140    @Override
1141    public boolean accept(Path p) {
1142      return HFileLink.isHFileLink(p);
1143    }
1144  }
1145
1146  public static class ReferenceFileFilter extends AbstractFileStatusFilter {
1147
1148    private final FileSystem fs;
1149
1150    public ReferenceFileFilter(FileSystem fs) {
1151      this.fs = fs;
1152    }
1153
1154    @Override
1155    protected boolean accept(Path p, @CheckForNull Boolean isDir) {
1156      if (!StoreFileInfo.isReference(p)) {
1157        return false;
1158      }
1159
1160      try {
1161        // only files can be references.
1162        return isFile(fs, isDir, p);
1163      } catch (IOException ioe) {
1164        // Maybe the file was moved or the fs was disconnected.
1165        LOG.warn("Skipping file {} due to IOException", p, ioe);
1166        return false;
1167      }
1168    }
1169  }
1170
1171  /**
1172   * Called every so-often by storefile map builder getTableStoreFilePathMap to report progress.
1173   */
1174  interface ProgressReporter {
1175    /**
1176     * @param status File or directory we are about to process.
1177     */
1178    void progress(FileStatus status);
1179  }
1180
1181  /**
1182   * Runs through the HBase rootdir/tablename and creates a reverse lookup map for table StoreFile
1183   * names to the full Path. <br>
1184   * Example...<br>
1185   * Key = 3944417774205889744 <br>
1186   * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
1187   * @param map          map to add values. If null, this method will create and populate one to
1188   *                     return
1189   * @param fs           The file system to use.
1190   * @param hbaseRootDir The root directory to scan.
1191   * @param tableName    name of the table to scan.
1192   * @return Map keyed by StoreFile name with a value of the full Path.
1193   * @throws IOException When scanning the directory fails.
1194   */
1195  public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> map,
1196    final FileSystem fs, final Path hbaseRootDir, TableName tableName)
1197    throws IOException, InterruptedException {
1198    return getTableStoreFilePathMap(map, fs, hbaseRootDir, tableName, null, null,
1199      (ProgressReporter) null);
1200  }
1201
1202  /**
1203   * Runs through the HBase rootdir/tablename and creates a reverse lookup map for table StoreFile
1204   * names to the full Path. Note that because this method can be called on a 'live' HBase system
1205   * that we will skip files that no longer exist by the time we traverse them and similarly the
1206   * user of the result needs to consider that some entries in this map may not exist by the time
1207   * this call completes. <br>
1208   * Example...<br>
1209   * Key = 3944417774205889744 <br>
1210   * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
1211   * @param resultMap        map to add values. If null, this method will create and populate one to
1212   *                         return
1213   * @param fs               The file system to use.
1214   * @param hbaseRootDir     The root directory to scan.
1215   * @param tableName        name of the table to scan.
1216   * @param sfFilter         optional path filter to apply to store files
1217   * @param executor         optional executor service to parallelize this operation
1218   * @param progressReporter Instance or null; gets called every time we move to new region of
1219   *                         family dir and for each store file.
1220   * @return Map keyed by StoreFile name with a value of the full Path.
1221   * @throws IOException When scanning the directory fails.
1222   * @deprecated Since 2.3.0. For removal in hbase4. Use ProgressReporter override instead.
1223   */
1224  @Deprecated
1225  public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> resultMap,
1226    final FileSystem fs, final Path hbaseRootDir, TableName tableName, final PathFilter sfFilter,
1227    ExecutorService executor, final HbckErrorReporter progressReporter)
1228    throws IOException, InterruptedException {
1229    return getTableStoreFilePathMap(resultMap, fs, hbaseRootDir, tableName, sfFilter, executor,
1230      new ProgressReporter() {
1231        @Override
1232        public void progress(FileStatus status) {
1233          // status is not used in this implementation.
1234          progressReporter.progress();
1235        }
1236      });
1237  }
1238
1239  /**
1240   * Runs through the HBase rootdir/tablename and creates a reverse lookup map for table StoreFile
1241   * names to the full Path. Note that because this method can be called on a 'live' HBase system
1242   * that we will skip files that no longer exist by the time we traverse them and similarly the
1243   * user of the result needs to consider that some entries in this map may not exist by the time
1244   * this call completes. <br>
1245   * Example...<br>
1246   * Key = 3944417774205889744 <br>
1247   * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
1248   * @param resultMap        map to add values. If null, this method will create and populate one to
1249   *                         return
1250   * @param fs               The file system to use.
1251   * @param hbaseRootDir     The root directory to scan.
1252   * @param tableName        name of the table to scan.
1253   * @param sfFilter         optional path filter to apply to store files
1254   * @param executor         optional executor service to parallelize this operation
1255   * @param progressReporter Instance or null; gets called every time we move to new region of
1256   *                         family dir and for each store file.
1257   * @return Map keyed by StoreFile name with a value of the full Path.
1258   * @throws IOException          When scanning the directory fails.
1259   * @throws InterruptedException the thread is interrupted, either before or during the activity.
1260   */
1261  public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> resultMap,
1262    final FileSystem fs, final Path hbaseRootDir, TableName tableName, final PathFilter sfFilter,
1263    ExecutorService executor, final ProgressReporter progressReporter)
1264    throws IOException, InterruptedException {
1265
1266    final Map<String, Path> finalResultMap =
1267      resultMap == null ? new ConcurrentHashMap<>(128, 0.75f, 32) : resultMap;
1268
1269    // only include the directory paths to tables
1270    Path tableDir = CommonFSUtils.getTableDir(hbaseRootDir, tableName);
1271    // Inside a table, there are compaction.dir directories to skip. Otherwise, all else
1272    // should be regions.
1273    final FamilyDirFilter familyFilter = new FamilyDirFilter(fs);
1274    final Vector<Exception> exceptions = new Vector<>();
1275
1276    try {
1277      List<FileStatus> regionDirs =
1278        FSUtils.listStatusWithStatusFilter(fs, tableDir, new RegionDirFilter(fs));
1279      if (regionDirs == null) {
1280        return finalResultMap;
1281      }
1282
1283      final List<Future<?>> futures = new ArrayList<>(regionDirs.size());
1284
1285      for (FileStatus regionDir : regionDirs) {
1286        if (null != progressReporter) {
1287          progressReporter.progress(regionDir);
1288        }
1289        final Path dd = regionDir.getPath();
1290
1291        if (!exceptions.isEmpty()) {
1292          break;
1293        }
1294
1295        Runnable getRegionStoreFileMapCall = new Runnable() {
1296          @Override
1297          public void run() {
1298            try {
1299              HashMap<String, Path> regionStoreFileMap = new HashMap<>();
1300              List<FileStatus> familyDirs =
1301                FSUtils.listStatusWithStatusFilter(fs, dd, familyFilter);
1302              if (familyDirs == null) {
1303                if (!fs.exists(dd)) {
1304                  LOG.warn("Skipping region because it no longer exists: " + dd);
1305                } else {
1306                  LOG.warn("Skipping region because it has no family dirs: " + dd);
1307                }
1308                return;
1309              }
1310              for (FileStatus familyDir : familyDirs) {
1311                if (null != progressReporter) {
1312                  progressReporter.progress(familyDir);
1313                }
1314                Path family = familyDir.getPath();
1315                if (family.getName().equals(HConstants.RECOVERED_EDITS_DIR)) {
1316                  continue;
1317                }
1318                // now in family, iterate over the StoreFiles and
1319                // put in map
1320                FileStatus[] familyStatus = fs.listStatus(family);
1321                for (FileStatus sfStatus : familyStatus) {
1322                  if (null != progressReporter) {
1323                    progressReporter.progress(sfStatus);
1324                  }
1325                  Path sf = sfStatus.getPath();
1326                  if (sfFilter == null || sfFilter.accept(sf)) {
1327                    regionStoreFileMap.put(sf.getName(), sf);
1328                  }
1329                }
1330              }
1331              finalResultMap.putAll(regionStoreFileMap);
1332            } catch (Exception e) {
1333              LOG.error("Could not get region store file map for region: " + dd, e);
1334              exceptions.add(e);
1335            }
1336          }
1337        };
1338
1339        // If executor is available, submit async tasks to exec concurrently, otherwise
1340        // just do serial sync execution
1341        if (executor != null) {
1342          Future<?> future = executor.submit(getRegionStoreFileMapCall);
1343          futures.add(future);
1344        } else {
1345          FutureTask<?> future = new FutureTask<>(getRegionStoreFileMapCall, null);
1346          future.run();
1347          futures.add(future);
1348        }
1349      }
1350
1351      // Ensure all pending tasks are complete (or that we run into an exception)
1352      for (Future<?> f : futures) {
1353        if (!exceptions.isEmpty()) {
1354          break;
1355        }
1356        try {
1357          f.get();
1358        } catch (ExecutionException e) {
1359          LOG.error("Unexpected exec exception!  Should've been caught already.  (Bug?)", e);
1360          // Shouldn't happen, we already logged/caught any exceptions in the Runnable
1361        }
1362      }
1363    } catch (IOException e) {
1364      LOG.error("Cannot execute getTableStoreFilePathMap for " + tableName, e);
1365      exceptions.add(e);
1366    } finally {
1367      if (!exceptions.isEmpty()) {
1368        // Just throw the first exception as an indication something bad happened
1369        // Don't need to propagate all the exceptions, we already logged them all anyway
1370        Throwables.propagateIfPossible(exceptions.firstElement(), IOException.class);
1371        throw new IOException(exceptions.firstElement());
1372      }
1373    }
1374
1375    return finalResultMap;
1376  }
1377
1378  public static int getRegionReferenceFileCount(final FileSystem fs, final Path p) {
1379    int result = 0;
1380    try {
1381      for (Path familyDir : getFamilyDirs(fs, p)) {
1382        result += getReferenceFilePaths(fs, familyDir).size();
1383      }
1384    } catch (IOException e) {
1385      LOG.warn("Error counting reference files", e);
1386    }
1387    return result;
1388  }
1389
1390  /**
1391   * Runs through the HBase rootdir and creates a reverse lookup map for table StoreFile names to
1392   * the full Path. <br>
1393   * Example...<br>
1394   * Key = 3944417774205889744 <br>
1395   * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
1396   * @param fs           The file system to use.
1397   * @param hbaseRootDir The root directory to scan.
1398   * @return Map keyed by StoreFile name with a value of the full Path.
1399   * @throws IOException When scanning the directory fails.
1400   */
1401  public static Map<String, Path> getTableStoreFilePathMap(final FileSystem fs,
1402    final Path hbaseRootDir) throws IOException, InterruptedException {
1403    return getTableStoreFilePathMap(fs, hbaseRootDir, null, null, (ProgressReporter) null);
1404  }
1405
1406  /**
1407   * Runs through the HBase rootdir and creates a reverse lookup map for table StoreFile names to
1408   * the full Path. <br>
1409   * Example...<br>
1410   * Key = 3944417774205889744 <br>
1411   * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
1412   * @param fs               The file system to use.
1413   * @param hbaseRootDir     The root directory to scan.
1414   * @param sfFilter         optional path filter to apply to store files
1415   * @param executor         optional executor service to parallelize this operation
1416   * @param progressReporter Instance or null; gets called every time we move to new region of
1417   *                         family dir and for each store file.
1418   * @return Map keyed by StoreFile name with a value of the full Path.
1419   * @throws IOException When scanning the directory fails.
1420   * @deprecated Since 2.3.0. Will be removed in hbase4. Used
1421   *             {@link #getTableStoreFilePathMap(FileSystem, Path, PathFilter, ExecutorService, ProgressReporter)}
1422   */
1423  @Deprecated
1424  public static Map<String, Path> getTableStoreFilePathMap(final FileSystem fs,
1425    final Path hbaseRootDir, PathFilter sfFilter, ExecutorService executor,
1426    HbckErrorReporter progressReporter) throws IOException, InterruptedException {
1427    return getTableStoreFilePathMap(fs, hbaseRootDir, sfFilter, executor, new ProgressReporter() {
1428      @Override
1429      public void progress(FileStatus status) {
1430        // status is not used in this implementation.
1431        progressReporter.progress();
1432      }
1433    });
1434  }
1435
1436  /**
1437   * Runs through the HBase rootdir and creates a reverse lookup map for table StoreFile names to
1438   * the full Path. <br>
1439   * Example...<br>
1440   * Key = 3944417774205889744 <br>
1441   * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
1442   * @param fs               The file system to use.
1443   * @param hbaseRootDir     The root directory to scan.
1444   * @param sfFilter         optional path filter to apply to store files
1445   * @param executor         optional executor service to parallelize this operation
1446   * @param progressReporter Instance or null; gets called every time we move to new region of
1447   *                         family dir and for each store file.
1448   * @return Map keyed by StoreFile name with a value of the full Path.
1449   * @throws IOException When scanning the directory fails.
1450   */
1451  public static Map<String, Path> getTableStoreFilePathMap(final FileSystem fs,
1452    final Path hbaseRootDir, PathFilter sfFilter, ExecutorService executor,
1453    ProgressReporter progressReporter) throws IOException, InterruptedException {
1454    ConcurrentHashMap<String, Path> map = new ConcurrentHashMap<>(1024, 0.75f, 32);
1455
1456    // if this method looks similar to 'getTableFragmentation' that is because
1457    // it was borrowed from it.
1458
1459    // only include the directory paths to tables
1460    for (Path tableDir : FSUtils.getTableDirs(fs, hbaseRootDir)) {
1461      getTableStoreFilePathMap(map, fs, hbaseRootDir, CommonFSUtils.getTableName(tableDir),
1462        sfFilter, executor, progressReporter);
1463    }
1464    return map;
1465  }
1466
1467  /**
1468   * Filters FileStatuses in an array and returns a list
1469   * @param input  An array of FileStatuses
1470   * @param filter A required filter to filter the array
1471   * @return A list of FileStatuses
1472   */
1473  public static List<FileStatus> filterFileStatuses(FileStatus[] input, FileStatusFilter filter) {
1474    if (input == null) return null;
1475    return filterFileStatuses(Iterators.forArray(input), filter);
1476  }
1477
1478  /**
1479   * Filters FileStatuses in an iterator and returns a list
1480   * @param input  An iterator of FileStatuses
1481   * @param filter A required filter to filter the array
1482   * @return A list of FileStatuses
1483   */
1484  public static List<FileStatus> filterFileStatuses(Iterator<FileStatus> input,
1485    FileStatusFilter filter) {
1486    if (input == null) return null;
1487    ArrayList<FileStatus> results = new ArrayList<>();
1488    while (input.hasNext()) {
1489      FileStatus f = input.next();
1490      if (filter.accept(f)) {
1491        results.add(f);
1492      }
1493    }
1494    return results;
1495  }
1496
1497  /**
1498   * Calls fs.listStatus() and treats FileNotFoundException as non-fatal This accommodates
1499   * differences between hadoop versions, where hadoop 1 does not throw a FileNotFoundException, and
1500   * return an empty FileStatus[] while Hadoop 2 will throw FileNotFoundException.
1501   * @param fs     file system
1502   * @param dir    directory
1503   * @param filter file status filter
1504   * @return null if dir is empty or doesn't exist, otherwise FileStatus list
1505   */
1506  public static List<FileStatus> listStatusWithStatusFilter(final FileSystem fs, final Path dir,
1507    final FileStatusFilter filter) throws IOException {
1508    FileStatus[] status = null;
1509    try {
1510      status = fs.listStatus(dir);
1511    } catch (FileNotFoundException fnfe) {
1512      LOG.trace("{} does not exist", dir);
1513      return null;
1514    }
1515
1516    if (ArrayUtils.getLength(status) == 0) {
1517      return null;
1518    }
1519
1520    if (filter == null) {
1521      return Arrays.asList(status);
1522    } else {
1523      List<FileStatus> status2 = filterFileStatuses(status, filter);
1524      if (status2 == null || status2.isEmpty()) {
1525        return null;
1526      } else {
1527        return status2;
1528      }
1529    }
1530  }
1531
1532  /**
1533   * This function is to scan the root path of the file system to get the degree of locality for
1534   * each region on each of the servers having at least one block of that region. This is used by
1535   * the tool {@link org.apache.hadoop.hbase.master.RegionPlacementMaintainer} the configuration to
1536   * use
1537   * @return the mapping from region encoded name to a map of server names to locality fraction in
1538   *         case of file system errors or interrupts
1539   */
1540  public static Map<String, Map<String, Float>>
1541    getRegionDegreeLocalityMappingFromFS(final Configuration conf) throws IOException {
1542    return getRegionDegreeLocalityMappingFromFS(conf, null,
1543      conf.getInt(THREAD_POOLSIZE, DEFAULT_THREAD_POOLSIZE));
1544
1545  }
1546
1547  /**
1548   * This function is to scan the root path of the file system to get the degree of locality for
1549   * each region on each of the servers having at least one block of that region. the configuration
1550   * to use the table you wish to scan locality for the thread pool size to use
1551   * @return the mapping from region encoded name to a map of server names to locality fraction in
1552   *         case of file system errors or interrupts
1553   */
1554  public static Map<String, Map<String, Float>> getRegionDegreeLocalityMappingFromFS(
1555    final Configuration conf, final String desiredTable, int threadPoolSize) throws IOException {
1556    Map<String, Map<String, Float>> regionDegreeLocalityMapping = new ConcurrentHashMap<>();
1557    getRegionLocalityMappingFromFS(conf, desiredTable, threadPoolSize, regionDegreeLocalityMapping);
1558    return regionDegreeLocalityMapping;
1559  }
1560
1561  /**
1562   * This function is to scan the root path of the file system to get either the mapping between the
1563   * region name and its best locality region server or the degree of locality of each region on
1564   * each of the servers having at least one block of that region. The output map parameters are
1565   * both optional. the configuration to use the table you wish to scan locality for the thread pool
1566   * size to use the map into which to put the locality degree mapping or null, must be a
1567   * thread-safe implementation in case of file system errors or interrupts
1568   */
1569  private static void getRegionLocalityMappingFromFS(final Configuration conf,
1570    final String desiredTable, int threadPoolSize,
1571    final Map<String, Map<String, Float>> regionDegreeLocalityMapping) throws IOException {
1572    final FileSystem fs = FileSystem.get(conf);
1573    final Path rootPath = CommonFSUtils.getRootDir(conf);
1574    final long startTime = EnvironmentEdgeManager.currentTime();
1575    final Path queryPath;
1576    // The table files are in ${hbase.rootdir}/data/<namespace>/<table>/*
1577    if (null == desiredTable) {
1578      queryPath =
1579        new Path(new Path(rootPath, HConstants.BASE_NAMESPACE_DIR).toString() + "/*/*/*/");
1580    } else {
1581      queryPath = new Path(
1582        CommonFSUtils.getTableDir(rootPath, TableName.valueOf(desiredTable)).toString() + "/*/");
1583    }
1584
1585    // reject all paths that are not appropriate
1586    PathFilter pathFilter = new PathFilter() {
1587      @Override
1588      public boolean accept(Path path) {
1589        // this is the region name; it may get some noise data
1590        if (null == path) {
1591          return false;
1592        }
1593
1594        // no parent?
1595        Path parent = path.getParent();
1596        if (null == parent) {
1597          return false;
1598        }
1599
1600        String regionName = path.getName();
1601        if (null == regionName) {
1602          return false;
1603        }
1604
1605        if (!regionName.toLowerCase(Locale.ROOT).matches("[0-9a-f]+")) {
1606          return false;
1607        }
1608        return true;
1609      }
1610    };
1611
1612    FileStatus[] statusList = fs.globStatus(queryPath, pathFilter);
1613
1614    if (LOG.isDebugEnabled()) {
1615      LOG.debug("Query Path: {} ; # list of files: {}", queryPath, Arrays.toString(statusList));
1616    }
1617
1618    if (null == statusList) {
1619      return;
1620    }
1621
1622    // lower the number of threads in case we have very few expected regions
1623    threadPoolSize = Math.min(threadPoolSize, statusList.length);
1624
1625    // run in multiple threads
1626    final ExecutorService tpe = Executors.newFixedThreadPool(threadPoolSize,
1627      new ThreadFactoryBuilder().setNameFormat("FSRegionQuery-pool-%d").setDaemon(true)
1628        .setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER).build());
1629    try {
1630      // ignore all file status items that are not of interest
1631      for (FileStatus regionStatus : statusList) {
1632        if (null == regionStatus || !regionStatus.isDirectory()) {
1633          continue;
1634        }
1635
1636        final Path regionPath = regionStatus.getPath();
1637        if (null != regionPath) {
1638          tpe.execute(new FSRegionScanner(fs, regionPath, null, regionDegreeLocalityMapping));
1639        }
1640      }
1641    } finally {
1642      tpe.shutdown();
1643      final long threadWakeFrequency = (long) conf.getInt(HConstants.THREAD_WAKE_FREQUENCY,
1644        HConstants.DEFAULT_THREAD_WAKE_FREQUENCY);
1645      try {
1646        // here we wait until TPE terminates, which is either naturally or by
1647        // exceptions in the execution of the threads
1648        while (!tpe.awaitTermination(threadWakeFrequency, TimeUnit.MILLISECONDS)) {
1649          // printing out rough estimate, so as to not introduce
1650          // AtomicInteger
1651          LOG.info("Locality checking is underway: { Scanned Regions : "
1652            + ((ThreadPoolExecutor) tpe).getCompletedTaskCount() + "/"
1653            + ((ThreadPoolExecutor) tpe).getTaskCount() + " }");
1654        }
1655      } catch (InterruptedException e) {
1656        Thread.currentThread().interrupt();
1657        throw (InterruptedIOException) new InterruptedIOException().initCause(e);
1658      }
1659    }
1660
1661    long overhead = EnvironmentEdgeManager.currentTime() - startTime;
1662    LOG.info("Scan DFS for locality info takes {}ms", overhead);
1663  }
1664
1665  /**
1666   * Do our short circuit read setup. Checks buffer size to use and whether to do checksumming in
1667   * hbase or hdfs.
1668   */
1669  public static void setupShortCircuitRead(final Configuration conf) {
1670    // Check that the user has not set the "dfs.client.read.shortcircuit.skip.checksum" property.
1671    boolean shortCircuitSkipChecksum =
1672      conf.getBoolean("dfs.client.read.shortcircuit.skip.checksum", false);
1673    boolean useHBaseChecksum = conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, true);
1674    if (shortCircuitSkipChecksum) {
1675      LOG.warn("Configuration \"dfs.client.read.shortcircuit.skip.checksum\" should not "
1676        + "be set to true."
1677        + (useHBaseChecksum
1678          ? " HBase checksum doesn't require "
1679            + "it, see https://issues.apache.org/jira/browse/HBASE-6868."
1680          : ""));
1681      assert !shortCircuitSkipChecksum; // this will fail if assertions are on
1682    }
1683    checkShortCircuitReadBufferSize(conf);
1684  }
1685
1686  /**
1687   * Check if short circuit read buffer size is set and if not, set it to hbase value.
1688   */
1689  public static void checkShortCircuitReadBufferSize(final Configuration conf) {
1690    final int defaultSize = HConstants.DEFAULT_BLOCKSIZE * 2;
1691    final int notSet = -1;
1692    // DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_BUFFER_SIZE_KEY is only defined in h2
1693    final String dfsKey = "dfs.client.read.shortcircuit.buffer.size";
1694    int size = conf.getInt(dfsKey, notSet);
1695    // If a size is set, return -- we will use it.
1696    if (size != notSet) return;
1697    // But short circuit buffer size is normally not set. Put in place the hbase wanted size.
1698    int hbaseSize = conf.getInt("hbase." + dfsKey, defaultSize);
1699    conf.setIfUnset(dfsKey, Integer.toString(hbaseSize));
1700  }
1701
1702  /**
1703   * Returns The DFSClient DFSHedgedReadMetrics instance or null if can't be found or not on hdfs.
1704   */
1705  public static DFSHedgedReadMetrics getDFSHedgedReadMetrics(final Configuration c)
1706    throws IOException {
1707    if (!CommonFSUtils.isHDFS(c)) {
1708      return null;
1709    }
1710    // getHedgedReadMetrics is package private. Get the DFSClient instance that is internal
1711    // to the DFS FS instance and make the method getHedgedReadMetrics accessible, then invoke it
1712    // to get the singleton instance of DFSHedgedReadMetrics shared by DFSClients.
1713    final String name = "getHedgedReadMetrics";
1714    DFSClient dfsclient = ((DistributedFileSystem) FileSystem.get(c)).getClient();
1715    Method m;
1716    try {
1717      m = dfsclient.getClass().getDeclaredMethod(name);
1718    } catch (NoSuchMethodException e) {
1719      LOG.warn(
1720        "Failed find method " + name + " in dfsclient; no hedged read metrics: " + e.getMessage());
1721      return null;
1722    } catch (SecurityException e) {
1723      LOG.warn(
1724        "Failed find method " + name + " in dfsclient; no hedged read metrics: " + e.getMessage());
1725      return null;
1726    }
1727    m.setAccessible(true);
1728    try {
1729      return (DFSHedgedReadMetrics) m.invoke(dfsclient);
1730    } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
1731      LOG.warn("Failed invoking method " + name + " on dfsclient; no hedged read metrics: "
1732        + e.getMessage());
1733      return null;
1734    }
1735  }
1736
1737  public static List<Path> copyFilesParallel(FileSystem srcFS, Path src, FileSystem dstFS, Path dst,
1738    Configuration conf, int threads) throws IOException {
1739    ExecutorService pool = Executors.newFixedThreadPool(threads);
1740    List<Future<Void>> futures = new ArrayList<>();
1741    List<Path> traversedPaths;
1742    try {
1743      traversedPaths = copyFiles(srcFS, src, dstFS, dst, conf, pool, futures);
1744      for (Future<Void> future : futures) {
1745        future.get();
1746      }
1747    } catch (ExecutionException | InterruptedException | IOException e) {
1748      throw new IOException("Copy snapshot reference files failed", e);
1749    } finally {
1750      pool.shutdownNow();
1751    }
1752    return traversedPaths;
1753  }
1754
1755  private static List<Path> copyFiles(FileSystem srcFS, Path src, FileSystem dstFS, Path dst,
1756    Configuration conf, ExecutorService pool, List<Future<Void>> futures) throws IOException {
1757    List<Path> traversedPaths = new ArrayList<>();
1758    traversedPaths.add(dst);
1759    FileStatus currentFileStatus = srcFS.getFileStatus(src);
1760    if (currentFileStatus.isDirectory()) {
1761      if (!dstFS.mkdirs(dst)) {
1762        throw new IOException("Create directory failed: " + dst);
1763      }
1764      FileStatus[] subPaths = srcFS.listStatus(src);
1765      for (FileStatus subPath : subPaths) {
1766        traversedPaths.addAll(copyFiles(srcFS, subPath.getPath(), dstFS,
1767          new Path(dst, subPath.getPath().getName()), conf, pool, futures));
1768      }
1769    } else {
1770      Future<Void> future = pool.submit(() -> {
1771        FileUtil.copy(srcFS, src, dstFS, dst, false, false, conf);
1772        return null;
1773      });
1774      futures.add(future);
1775    }
1776    return traversedPaths;
1777  }
1778
1779  /** Returns A set containing all namenode addresses of fs */
1780  private static Set<InetSocketAddress> getNNAddresses(DistributedFileSystem fs,
1781    Configuration conf) {
1782    Set<InetSocketAddress> addresses = new HashSet<>();
1783    String serviceName = fs.getCanonicalServiceName();
1784
1785    if (serviceName.startsWith("ha-hdfs")) {
1786      try {
1787        Map<String, Map<String, InetSocketAddress>> addressMap =
1788          DFSUtil.getNNServiceRpcAddressesForCluster(conf);
1789        String nameService = serviceName.substring(serviceName.indexOf(":") + 1);
1790        if (addressMap.containsKey(nameService)) {
1791          Map<String, InetSocketAddress> nnMap = addressMap.get(nameService);
1792          for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) {
1793            InetSocketAddress addr = e2.getValue();
1794            addresses.add(addr);
1795          }
1796        }
1797      } catch (Exception e) {
1798        LOG.warn("DFSUtil.getNNServiceRpcAddresses failed. serviceName=" + serviceName, e);
1799      }
1800    } else {
1801      URI uri = fs.getUri();
1802      int port = uri.getPort();
1803      if (port < 0) {
1804        int idx = serviceName.indexOf(':');
1805        port = Integer.parseInt(serviceName.substring(idx + 1));
1806      }
1807      InetSocketAddress addr = new InetSocketAddress(uri.getHost(), port);
1808      addresses.add(addr);
1809    }
1810
1811    return addresses;
1812  }
1813
1814  /**
1815   * @param conf the Configuration of HBase
1816   * @return Whether srcFs and desFs are on same hdfs or not
1817   */
1818  public static boolean isSameHdfs(Configuration conf, FileSystem srcFs, FileSystem desFs) {
1819    // By getCanonicalServiceName, we could make sure both srcFs and desFs
1820    // show a unified format which contains scheme, host and port.
1821    String srcServiceName = srcFs.getCanonicalServiceName();
1822    String desServiceName = desFs.getCanonicalServiceName();
1823
1824    if (srcServiceName == null || desServiceName == null) {
1825      return false;
1826    }
1827    if (srcServiceName.equals(desServiceName)) {
1828      return true;
1829    }
1830    if (srcServiceName.startsWith("ha-hdfs") && desServiceName.startsWith("ha-hdfs")) {
1831      Collection<String> internalNameServices =
1832        conf.getTrimmedStringCollection("dfs.internal.nameservices");
1833      if (!internalNameServices.isEmpty()) {
1834        if (internalNameServices.contains(srcServiceName.split(":")[1])) {
1835          return true;
1836        } else {
1837          return false;
1838        }
1839      }
1840    }
1841    if (srcFs instanceof DistributedFileSystem && desFs instanceof DistributedFileSystem) {
1842      // If one serviceName is an HA format while the other is a non-HA format,
1843      // maybe they refer to the same FileSystem.
1844      // For example, srcFs is "ha-hdfs://nameservices" and desFs is "hdfs://activeNamenode:port"
1845      Set<InetSocketAddress> srcAddrs = getNNAddresses((DistributedFileSystem) srcFs, conf);
1846      Set<InetSocketAddress> desAddrs = getNNAddresses((DistributedFileSystem) desFs, conf);
1847      if (Sets.intersection(srcAddrs, desAddrs).size() > 0) {
1848        return true;
1849      }
1850    }
1851
1852    return false;
1853  }
1854}