001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.io.HFileLink.LINK_NAME_PATTERN;
021
022import edu.umd.cs.findbugs.annotations.Nullable;
023import java.io.FileNotFoundException;
024import java.io.IOException;
025import java.io.InterruptedIOException;
026import java.util.ArrayList;
027import java.util.Collection;
028import java.util.HashMap;
029import java.util.List;
030import java.util.Map;
031import java.util.Objects;
032import java.util.Optional;
033import java.util.UUID;
034import java.util.regex.Matcher;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.fs.FSDataInputStream;
037import org.apache.hadoop.fs.FSDataOutputStream;
038import org.apache.hadoop.fs.FileStatus;
039import org.apache.hadoop.fs.FileSystem;
040import org.apache.hadoop.fs.FileUtil;
041import org.apache.hadoop.fs.LocatedFileStatus;
042import org.apache.hadoop.fs.Path;
043import org.apache.hadoop.fs.permission.FsPermission;
044import org.apache.hadoop.hbase.Cell;
045import org.apache.hadoop.hbase.HConstants;
046import org.apache.hadoop.hbase.PrivateCellUtil;
047import org.apache.hadoop.hbase.TableName;
048import org.apache.hadoop.hbase.backup.HFileArchiver;
049import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
050import org.apache.hadoop.hbase.client.RegionInfo;
051import org.apache.hadoop.hbase.client.TableDescriptor;
052import org.apache.hadoop.hbase.fs.HFileSystem;
053import org.apache.hadoop.hbase.io.HFileLink;
054import org.apache.hadoop.hbase.io.Reference;
055import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
056import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
057import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
058import org.apache.hadoop.hbase.util.Bytes;
059import org.apache.hadoop.hbase.util.CommonFSUtils;
060import org.apache.hadoop.hbase.util.FSUtils;
061import org.apache.hadoop.hbase.util.Pair;
062import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
063import org.apache.yetus.audience.InterfaceAudience;
064import org.slf4j.Logger;
065import org.slf4j.LoggerFactory;
066
067import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
068
069/**
070 * View to an on-disk Region. Provides the set of methods necessary to interact with the on-disk
071 * region data.
072 */
073@InterfaceAudience.Private
074public class HRegionFileSystem {
075  private static final Logger LOG = LoggerFactory.getLogger(HRegionFileSystem.class);
076
077  /** Name of the region info file that resides just under the region directory. */
078  public final static String REGION_INFO_FILE = ".regioninfo";
079
080  /** Temporary subdirectory of the region directory used for merges. */
081  public static final String REGION_MERGES_DIR = ".merges";
082
083  /** Temporary subdirectory of the region directory used for splits. */
084  public static final String REGION_SPLITS_DIR = ".splits";
085
086  /** Temporary subdirectory of the region directory used for compaction output. */
087  static final String REGION_TEMP_DIR = ".tmp";
088
089  private final RegionInfo regionInfo;
090  // regionInfo for interacting with FS (getting encodedName, etc)
091  final RegionInfo regionInfoForFs;
092  final Configuration conf;
093  private final Path tableDir;
094  final FileSystem fs;
095  private final Path regionDir;
096
097  /**
098   * In order to handle NN connectivity hiccups, one need to retry non-idempotent operation at the
099   * client level.
100   */
101  private final int hdfsClientRetriesNumber;
102  private final int baseSleepBeforeRetries;
103  private static final int DEFAULT_HDFS_CLIENT_RETRIES_NUMBER = 10;
104  private static final int DEFAULT_BASE_SLEEP_BEFORE_RETRIES = 1000;
105
106  /**
107   * Create a view to the on-disk region
108   * @param conf       the {@link Configuration} to use
109   * @param fs         {@link FileSystem} that contains the region
110   * @param tableDir   {@link Path} to where the table is being stored
111   * @param regionInfo {@link RegionInfo} for region
112   */
113  HRegionFileSystem(final Configuration conf, final FileSystem fs, final Path tableDir,
114    final RegionInfo regionInfo) {
115    this.fs = fs;
116    this.conf = conf;
117    this.tableDir = Objects.requireNonNull(tableDir, "tableDir is null");
118    this.regionInfo = Objects.requireNonNull(regionInfo, "regionInfo is null");
119    this.regionInfoForFs = ServerRegionReplicaUtil.getRegionInfoForFs(regionInfo);
120    this.regionDir = FSUtils.getRegionDirFromTableDir(tableDir, regionInfo);
121    this.hdfsClientRetriesNumber =
122      conf.getInt("hdfs.client.retries.number", DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
123    this.baseSleepBeforeRetries =
124      conf.getInt("hdfs.client.sleep.before.retries", DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
125  }
126
127  /** Returns the underlying {@link FileSystem} */
128  public FileSystem getFileSystem() {
129    return this.fs;
130  }
131
132  /** Returns the {@link RegionInfo} that describe this on-disk region view */
133  public RegionInfo getRegionInfo() {
134    return this.regionInfo;
135  }
136
137  public RegionInfo getRegionInfoForFS() {
138    return this.regionInfoForFs;
139  }
140
141  /** Returns {@link Path} to the region's root directory. */
142  public Path getTableDir() {
143    return this.tableDir;
144  }
145
146  /** Returns {@link Path} to the region directory. */
147  public Path getRegionDir() {
148    return regionDir;
149  }
150
151  // ===========================================================================
152  // Temp Helpers
153  // ===========================================================================
154  /** Returns {@link Path} to the region's temp directory, used for file creations */
155  public Path getTempDir() {
156    return new Path(getRegionDir(), REGION_TEMP_DIR);
157  }
158
159  /**
160   * Clean up any temp detritus that may have been left around from previous operation attempts.
161   */
162  void cleanupTempDir() throws IOException {
163    deleteDir(getTempDir());
164  }
165
166  // ===========================================================================
167  // Store/StoreFile Helpers
168  // ===========================================================================
169  /**
170   * Returns the directory path of the specified family
171   * @param familyName Column Family Name
172   * @return {@link Path} to the directory of the specified family
173   */
174  public Path getStoreDir(final String familyName) {
175    return new Path(this.getRegionDir(), familyName);
176  }
177
178  /**
179   * @param tabledir {@link Path} to where the table is being stored
180   * @param hri      {@link RegionInfo} for the region.
181   * @param family   {@link ColumnFamilyDescriptor} describing the column family
182   * @return Path to family/Store home directory.
183   */
184  public static Path getStoreHomedir(final Path tabledir, final RegionInfo hri,
185    final byte[] family) {
186    return getStoreHomedir(tabledir, hri.getEncodedName(), family);
187  }
188
189  /**
190   * @param tabledir    {@link Path} to where the table is being stored
191   * @param encodedName Encoded region name.
192   * @param family      {@link ColumnFamilyDescriptor} describing the column family
193   * @return Path to family/Store home directory.
194   */
195  public static Path getStoreHomedir(final Path tabledir, final String encodedName,
196    final byte[] family) {
197    return new Path(tabledir, new Path(encodedName, Bytes.toString(family)));
198  }
199
200  /**
201   * Create the store directory for the specified family name
202   * @param familyName Column Family Name
203   * @return {@link Path} to the directory of the specified family
204   * @throws IOException if the directory creation fails.
205   */
206  Path createStoreDir(final String familyName) throws IOException {
207    Path storeDir = getStoreDir(familyName);
208    if (!fs.exists(storeDir) && !createDir(storeDir))
209      throw new IOException("Failed creating " + storeDir);
210    return storeDir;
211  }
212
213  /**
214   * Set the directory of CF to the specified storage policy. <br>
215   * <i>"LAZY_PERSIST"</i>, <i>"ALL_SSD"</i>, <i>"ONE_SSD"</i>, <i>"HOT"</i>, <i>"WARM"</i>,
216   * <i>"COLD"</i> <br>
217   * <br>
218   * See {@link org.apache.hadoop.hdfs.protocol.HdfsConstants} for more details.
219   * @param familyName The name of column family.
220   * @param policyName The name of the storage policy: 'HOT', 'COLD', etc. See hadoop 2.6+
221   *                   org.apache.hadoop.hdfs.protocol.HdfsConstants for possible list e.g 'COLD',
222   *                   'WARM', 'HOT', 'ONE_SSD', 'ALL_SSD', 'LAZY_PERSIST'.
223   */
224  public void setStoragePolicy(String familyName, String policyName) {
225    CommonFSUtils.setStoragePolicy(this.fs, getStoreDir(familyName), policyName);
226  }
227
228  /**
229   * Set storage policy for a whole region. <br>
230   * <i>"LAZY_PERSIST"</i>, <i>"ALL_SSD"</i>, <i>"ONE_SSD"</i>, <i>"HOT"</i>, <i>"WARM"</i>,
231   * <i>"COLD"</i> <br>
232   * <br>
233   * See {@link org.apache.hadoop.hdfs.protocol.HdfsConstants} for more details.
234   * @param policyName The name of the storage policy: 'HOT', 'COLD', etc. See hadoop 2.6+
235   *                   org.apache.hadoop.hdfs.protocol.HdfsConstants for possible list e.g 'COLD',
236   *                   'WARM', 'HOT', 'ONE_SSD', 'ALL_SSD', 'LAZY_PERSIST'.
237   */
238  public void setStoragePolicy(String policyName) {
239    CommonFSUtils.setStoragePolicy(this.fs, getRegionDir(), policyName);
240  }
241
242  /**
243   * Get the storage policy of the directory of CF.
244   * @param familyName The name of column family.
245   * @return Storage policy name, or {@code null} if not using {@link HFileSystem} or exception
246   *         thrown when trying to get policy
247   */
248  @Nullable
249  public String getStoragePolicyName(String familyName) {
250    if (this.fs instanceof HFileSystem) {
251      Path storeDir = getStoreDir(familyName);
252      return ((HFileSystem) this.fs).getStoragePolicyName(storeDir);
253    }
254
255    return null;
256  }
257
258  /**
259   * Returns the store files available for the family. This methods performs the filtering based on
260   * the valid store files.
261   * @param familyName Column Family Name
262   * @return a set of {@link StoreFileInfo} for the specified family.
263   */
264  public List<StoreFileInfo> getStoreFiles(final String familyName) throws IOException {
265    return getStoreFiles(familyName, true);
266  }
267
268  /**
269   * Returns the store files available for the family. This methods performs the filtering based on
270   * the valid store files.
271   * @param familyName Column Family Name
272   * @return a set of {@link StoreFileInfo} for the specified family.
273   */
274  public List<StoreFileInfo> getStoreFiles(final String familyName, final boolean validate)
275    throws IOException {
276    Path familyDir = getStoreDir(familyName);
277    FileStatus[] files = CommonFSUtils.listStatus(this.fs, familyDir);
278    if (files == null) {
279      if (LOG.isTraceEnabled()) {
280        LOG.trace("No StoreFiles for: " + familyDir);
281      }
282      return null;
283    }
284
285    ArrayList<StoreFileInfo> storeFiles = new ArrayList<>(files.length);
286    for (FileStatus status : files) {
287      if (validate && !StoreFileInfo.isValid(status)) {
288        // recovered.hfiles directory is expected inside CF path when hbase.wal.split.to.hfile to
289        // true, refer HBASE-23740
290        if (!HConstants.RECOVERED_HFILES_DIR.equals(status.getPath().getName())) {
291          LOG.warn("Invalid StoreFile: {}", status.getPath());
292        }
293        continue;
294      }
295      StoreFileInfo info = ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, regionInfo,
296        regionInfoForFs, familyName, status.getPath());
297      storeFiles.add(info);
298
299    }
300    return storeFiles;
301  }
302
303  /**
304   * Returns the store files' LocatedFileStatus which available for the family. This methods
305   * performs the filtering based on the valid store files.
306   * @param familyName Column Family Name
307   * @return a list of store files' LocatedFileStatus for the specified family.
308   */
309  public static List<LocatedFileStatus> getStoreFilesLocatedStatus(final HRegionFileSystem regionfs,
310    final String familyName, final boolean validate) throws IOException {
311    Path familyDir = regionfs.getStoreDir(familyName);
312    List<LocatedFileStatus> locatedFileStatuses =
313      CommonFSUtils.listLocatedStatus(regionfs.getFileSystem(), familyDir);
314    if (locatedFileStatuses == null) {
315      if (LOG.isTraceEnabled()) {
316        LOG.trace("No StoreFiles for: " + familyDir);
317      }
318      return null;
319    }
320
321    List<LocatedFileStatus> validStoreFiles = Lists.newArrayList();
322    for (LocatedFileStatus status : locatedFileStatuses) {
323      if (validate && !StoreFileInfo.isValid(status)) {
324        // recovered.hfiles directory is expected inside CF path when hbase.wal.split.to.hfile to
325        // true, refer HBASE-23740
326        if (!HConstants.RECOVERED_HFILES_DIR.equals(status.getPath().getName())) {
327          LOG.warn("Invalid StoreFile: {}", status.getPath());
328        }
329      } else {
330        validStoreFiles.add(status);
331      }
332    }
333    return validStoreFiles;
334  }
335
336  /**
337   * Return Qualified Path of the specified family/file
338   * @param familyName Column Family Name
339   * @param fileName   File Name
340   * @return The qualified Path for the specified family/file
341   */
342  Path getStoreFilePath(final String familyName, final String fileName) {
343    Path familyDir = getStoreDir(familyName);
344    return new Path(familyDir, fileName).makeQualified(fs.getUri(), fs.getWorkingDirectory());
345  }
346
347  /**
348   * Return the store file information of the specified family/file.
349   * @param familyName Column Family Name
350   * @param fileName   File Name
351   * @return The {@link StoreFileInfo} for the specified family/file
352   */
353  StoreFileInfo getStoreFileInfo(final String familyName, final String fileName)
354    throws IOException {
355    Path familyDir = getStoreDir(familyName);
356    return ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, regionInfo, regionInfoForFs,
357      familyName, new Path(familyDir, fileName));
358  }
359
360  /**
361   * Returns true if the specified family has reference files
362   * @param familyName Column Family Name
363   * @return true if family contains reference files n
364   */
365  public boolean hasReferences(final String familyName) throws IOException {
366    Path storeDir = getStoreDir(familyName);
367    FileStatus[] files = CommonFSUtils.listStatus(fs, storeDir);
368    if (files != null) {
369      for (FileStatus stat : files) {
370        if (stat.isDirectory()) {
371          continue;
372        }
373        if (StoreFileInfo.isReference(stat.getPath())) {
374          LOG.trace("Reference {}", stat.getPath());
375          return true;
376        }
377      }
378    }
379    return false;
380  }
381
382  /**
383   * Check whether region has Reference file
384   * @param htd table desciptor of the region
385   * @return true if region has reference file n
386   */
387  public boolean hasReferences(final TableDescriptor htd) throws IOException {
388    for (ColumnFamilyDescriptor family : htd.getColumnFamilies()) {
389      if (hasReferences(family.getNameAsString())) {
390        return true;
391      }
392    }
393    return false;
394  }
395
396  /** Returns the set of families present on disk n */
397  public Collection<String> getFamilies() throws IOException {
398    FileStatus[] fds =
399      CommonFSUtils.listStatus(fs, getRegionDir(), new FSUtils.FamilyDirFilter(fs));
400    if (fds == null) return null;
401
402    ArrayList<String> families = new ArrayList<>(fds.length);
403    for (FileStatus status : fds) {
404      families.add(status.getPath().getName());
405    }
406
407    return families;
408  }
409
410  /**
411   * Remove the region family from disk, archiving the store files.
412   * @param familyName Column Family Name
413   * @throws IOException if an error occours during the archiving
414   */
415  public void deleteFamily(final String familyName) throws IOException {
416    // archive family store files
417    HFileArchiver.archiveFamily(fs, conf, regionInfoForFs, tableDir, Bytes.toBytes(familyName));
418
419    // delete the family folder
420    Path familyDir = getStoreDir(familyName);
421    if (fs.exists(familyDir) && !deleteDir(familyDir))
422      throw new IOException("Could not delete family " + familyName + " from FileSystem for region "
423        + regionInfoForFs.getRegionNameAsString() + "(" + regionInfoForFs.getEncodedName() + ")");
424  }
425
426  /**
427   * Generate a unique file name, used by createTempName() and commitStoreFile()
428   * @param suffix extra information to append to the generated name
429   * @return Unique file name
430   */
431  private static String generateUniqueName(final String suffix) {
432    String name = UUID.randomUUID().toString().replaceAll("-", "");
433    if (suffix != null) name += suffix;
434    return name;
435  }
436
437  /**
438   * Generate a unique temporary Path. Used in conjuction with commitStoreFile() to get a safer file
439   * creation. <code>
440   * Path file = fs.createTempName();
441   * ...StoreFile.Writer(file)...
442   * fs.commitStoreFile("family", file);
443   * </code>
444   * @return Unique {@link Path} of the temporary file
445   */
446  public Path createTempName() {
447    return createTempName(null);
448  }
449
450  /**
451   * Generate a unique temporary Path. Used in conjuction with commitStoreFile() to get a safer file
452   * creation. <code>
453   * Path file = fs.createTempName();
454   * ...StoreFile.Writer(file)...
455   * fs.commitStoreFile("family", file);
456   * </code>
457   * @param suffix extra information to append to the generated name
458   * @return Unique {@link Path} of the temporary file
459   */
460  public Path createTempName(final String suffix) {
461    return new Path(getTempDir(), generateUniqueName(suffix));
462  }
463
464  /**
465   * Move the file from a build/temp location to the main family store directory.
466   * @param familyName Family that will gain the file
467   * @param buildPath  {@link Path} to the file to commit.
468   * @return The new {@link Path} of the committed file n
469   */
470  public Path commitStoreFile(final String familyName, final Path buildPath) throws IOException {
471    Path dstPath = preCommitStoreFile(familyName, buildPath, -1, false);
472    return commitStoreFile(buildPath, dstPath);
473  }
474
475  /**
476   * Generate the filename in the main family store directory for moving the file from a build/temp
477   * location.
478   * @param familyName      Family that will gain the file
479   * @param buildPath       {@link Path} to the file to commit.
480   * @param seqNum          Sequence Number to append to the file name (less then 0 if no sequence
481   *                        number)
482   * @param generateNewName False if you want to keep the buildPath name
483   * @return The new {@link Path} of the to be committed file n
484   */
485  private Path preCommitStoreFile(final String familyName, final Path buildPath, final long seqNum,
486    final boolean generateNewName) throws IOException {
487    Path storeDir = getStoreDir(familyName);
488    if (!fs.exists(storeDir) && !createDir(storeDir))
489      throw new IOException("Failed creating " + storeDir);
490
491    String name = buildPath.getName();
492    if (generateNewName) {
493      name = generateUniqueName((seqNum < 0) ? null : "_SeqId_" + seqNum + "_");
494    }
495    Path dstPath = new Path(storeDir, name);
496    if (!fs.exists(buildPath)) {
497      throw new FileNotFoundException(buildPath.toString());
498    }
499    if (LOG.isDebugEnabled()) {
500      LOG.debug("Committing " + buildPath + " as " + dstPath);
501    }
502    return dstPath;
503  }
504
505  /*
506   * Moves file from staging dir to region dir
507   * @param buildPath {@link Path} to the file to commit.
508   * @param dstPath {@link Path} to the file under region dir
509   * @return The {@link Path} of the committed file n
510   */
511  Path commitStoreFile(final Path buildPath, Path dstPath) throws IOException {
512    // rename is not necessary in case of direct-insert stores
513    if (buildPath.equals(dstPath)) {
514      return dstPath;
515    }
516    // buildPath exists, therefore not doing an exists() check.
517    if (!rename(buildPath, dstPath)) {
518      throw new IOException("Failed rename of " + buildPath + " to " + dstPath);
519    }
520    return dstPath;
521  }
522
523  /**
524   * Archives the specified store file from the specified family.
525   * @param familyName Family that contains the store files
526   * @param filePath   {@link Path} to the store file to remove
527   * @throws IOException if the archiving fails
528   */
529  public void removeStoreFile(final String familyName, final Path filePath) throws IOException {
530    HFileArchiver.archiveStoreFile(this.conf, this.fs, this.regionInfoForFs, this.tableDir,
531      Bytes.toBytes(familyName), filePath);
532  }
533
534  /**
535   * Closes and archives the specified store files from the specified family.
536   * @param familyName Family that contains the store files
537   * @param storeFiles set of store files to remove
538   * @throws IOException if the archiving fails
539   */
540  public void removeStoreFiles(String familyName, Collection<HStoreFile> storeFiles)
541    throws IOException {
542    HFileArchiver.archiveStoreFiles(this.conf, this.fs, this.regionInfoForFs, this.tableDir,
543      Bytes.toBytes(familyName), storeFiles);
544  }
545
546  /**
547   * Bulk load: Add a specified store file to the specified family. If the source file is on the
548   * same different file-system is moved from the source location to the destination location,
549   * otherwise is copied over.
550   * @param familyName Family that will gain the file
551   * @param srcPath    {@link Path} to the file to import
552   * @param seqNum     Bulk Load sequence number
553   * @return The destination {@link Path} of the bulk loaded file n
554   */
555  Pair<Path, Path> bulkLoadStoreFile(final String familyName, Path srcPath, long seqNum)
556    throws IOException {
557    // Copy the file if it's on another filesystem
558    FileSystem srcFs = srcPath.getFileSystem(conf);
559    srcPath = srcFs.resolvePath(srcPath);
560    FileSystem realSrcFs = srcPath.getFileSystem(conf);
561    FileSystem desFs = fs instanceof HFileSystem ? ((HFileSystem) fs).getBackingFs() : fs;
562
563    // We can't compare FileSystem instances as equals() includes UGI instance
564    // as part of the comparison and won't work when doing SecureBulkLoad
565    // TODO deal with viewFS
566    if (!FSUtils.isSameHdfs(conf, realSrcFs, desFs)) {
567      LOG.info("Bulk-load file " + srcPath + " is on different filesystem than "
568        + "the destination store. Copying file over to destination filesystem.");
569      Path tmpPath = createTempName();
570      FileUtil.copy(realSrcFs, srcPath, fs, tmpPath, false, conf);
571      LOG.info("Copied " + srcPath + " to temporary path on destination filesystem: " + tmpPath);
572      srcPath = tmpPath;
573    }
574
575    return new Pair<>(srcPath, preCommitStoreFile(familyName, srcPath, seqNum, true));
576  }
577
578  // ===========================================================================
579  // Splits Helpers
580  // ===========================================================================
581
582  public Path getSplitsDir(final RegionInfo hri) {
583    return new Path(getTableDir(), hri.getEncodedName());
584  }
585
586  /**
587   * Remove daughter region
588   * @param regionInfo daughter {@link RegionInfo} n
589   */
590  void cleanupDaughterRegion(final RegionInfo regionInfo) throws IOException {
591    Path regionDir = new Path(this.tableDir, regionInfo.getEncodedName());
592    if (this.fs.exists(regionDir) && !deleteDir(regionDir)) {
593      throw new IOException("Failed delete of " + regionDir);
594    }
595  }
596
597  /**
598   * Commit a daughter region, moving it from the split temporary directory to the proper location
599   * in the filesystem.
600   * @param regionInfo daughter {@link org.apache.hadoop.hbase.client.RegionInfo}
601   */
602  public Path commitDaughterRegion(final RegionInfo regionInfo, List<Path> allRegionFiles,
603    MasterProcedureEnv env) throws IOException {
604    Path regionDir = this.getSplitsDir(regionInfo);
605    if (fs.exists(regionDir)) {
606      // Write HRI to a file in case we need to recover hbase:meta
607      Path regionInfoFile = new Path(regionDir, REGION_INFO_FILE);
608      byte[] regionInfoContent = getRegionInfoFileContent(regionInfo);
609      writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
610      HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
611        env.getMasterConfiguration(), fs, getTableDir(), regionInfo, false);
612      insertRegionFilesIntoStoreTracker(allRegionFiles, env, regionFs);
613    }
614    return regionDir;
615  }
616
617  private void insertRegionFilesIntoStoreTracker(List<Path> allFiles, MasterProcedureEnv env,
618    HRegionFileSystem regionFs) throws IOException {
619    TableDescriptor tblDesc =
620      env.getMasterServices().getTableDescriptors().get(regionInfo.getTable());
621    // we need to map trackers per store
622    Map<String, StoreFileTracker> trackerMap = new HashMap<>();
623    // we need to map store files per store
624    Map<String, List<StoreFileInfo>> fileInfoMap = new HashMap<>();
625    for (Path file : allFiles) {
626      String familyName = file.getParent().getName();
627      trackerMap.computeIfAbsent(familyName, t -> StoreFileTrackerFactory.create(conf, tblDesc,
628        tblDesc.getColumnFamily(Bytes.toBytes(familyName)), regionFs));
629      fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>());
630      List<StoreFileInfo> infos = fileInfoMap.get(familyName);
631      infos.add(new StoreFileInfo(conf, fs, file, true));
632    }
633    for (Map.Entry<String, StoreFileTracker> entry : trackerMap.entrySet()) {
634      entry.getValue().add(fileInfoMap.get(entry.getKey()));
635    }
636  }
637
638  /**
639   * Creates region split daughter directories under the table dir. If the daughter regions already
640   * exist, for example, in the case of a recovery from a previous failed split procedure, this
641   * method deletes the given region dir recursively, then recreates it again.
642   */
643  public void createSplitsDir(RegionInfo daughterA, RegionInfo daughterB) throws IOException {
644    Path daughterADir = getSplitsDir(daughterA);
645    if (fs.exists(daughterADir) && !deleteDir(daughterADir)) {
646      throw new IOException("Failed deletion of " + daughterADir + " before creating them again.");
647
648    }
649    if (!createDir(daughterADir)) {
650      throw new IOException("Failed create of " + daughterADir);
651    }
652    Path daughterBDir = getSplitsDir(daughterB);
653    if (fs.exists(daughterBDir) && !deleteDir(daughterBDir)) {
654      throw new IOException("Failed deletion of " + daughterBDir + " before creating them again.");
655
656    }
657    if (!createDir(daughterBDir)) {
658      throw new IOException("Failed create of " + daughterBDir);
659    }
660  }
661
662  /**
663   * Write out a split reference. Package local so it doesnt leak out of regionserver.
664   * @param hri         {@link RegionInfo} of the destination
665   * @param familyName  Column Family Name
666   * @param f           File to split.
667   * @param splitRow    Split Row
668   * @param top         True if we are referring to the top half of the hfile.
669   * @param splitPolicy A split policy instance; be careful! May not be full populated; e.g. if this
670   *                    method is invoked on the Master side, then the RegionSplitPolicy will NOT
671   *                    have a reference to a Region.
672   * @return Path to created reference.
673   */
674  public Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, byte[] splitRow,
675    boolean top, RegionSplitPolicy splitPolicy) throws IOException {
676    Path splitDir = new Path(getSplitsDir(hri), familyName);
677    // Add the referred-to regions name as a dot separated suffix.
678    // See REF_NAME_REGEX regex above. The referred-to regions name is
679    // up in the path of the passed in <code>f</code> -- parentdir is family,
680    // then the directory above is the region name.
681    String parentRegionName = regionInfoForFs.getEncodedName();
682    // Write reference with same file id only with the other region name as
683    // suffix and into the new region location (under same family).
684    Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
685    if (fs.exists(p)) {
686      LOG.warn("Found an already existing split file for {}. Assuming this is a recovery.", p);
687      return p;
688    }
689    boolean createLinkFile = false;
690    if (splitPolicy == null || !splitPolicy.skipStoreFileRangeCheck(familyName)) {
691      // Check whether the split row lies in the range of the store file
692      // If it is outside the range, return directly.
693      f.initReader();
694      try {
695        Cell splitKey = PrivateCellUtil.createFirstOnRow(splitRow);
696        Optional<Cell> lastKey = f.getLastKey();
697        Optional<Cell> firstKey = f.getFirstKey();
698        if (top) {
699          // check if larger than last key.
700          // If lastKey is null means storefile is empty.
701          if (!lastKey.isPresent()) {
702            return null;
703          }
704          if (f.getComparator().compare(splitKey, lastKey.get()) > 0) {
705            return null;
706          }
707          if (firstKey.isPresent() && f.getComparator().compare(splitKey, firstKey.get()) <= 0) {
708            LOG.debug("Will create HFileLink file for {}, top=true", f.getPath());
709            createLinkFile = true;
710          }
711        } else {
712          // check if smaller than first key
713          // If firstKey is null means storefile is empty.
714          if (!firstKey.isPresent()) {
715            return null;
716          }
717          if (f.getComparator().compare(splitKey, firstKey.get()) < 0) {
718            return null;
719          }
720          if (lastKey.isPresent() && f.getComparator().compare(splitKey, lastKey.get()) >= 0) {
721            LOG.debug("Will create HFileLink file for {}, top=false", f.getPath());
722            createLinkFile = true;
723          }
724        }
725      } finally {
726        f.closeStoreFile(f.getCacheConf() != null ? f.getCacheConf().shouldEvictOnClose() : true);
727      }
728    }
729    if (createLinkFile) {
730      // create HFileLink file instead of Reference file for child
731      String hfileName = f.getPath().getName();
732      TableName linkedTable = regionInfoForFs.getTable();
733      String linkedRegion = regionInfoForFs.getEncodedName();
734      try {
735        if (HFileLink.isHFileLink(hfileName)) {
736          Matcher m = LINK_NAME_PATTERN.matcher(hfileName);
737          if (!m.matches()) {
738            throw new IllegalArgumentException(hfileName + " is not a valid HFileLink name!");
739          }
740          linkedTable = TableName.valueOf(m.group(1), m.group(2));
741          linkedRegion = m.group(3);
742          hfileName = m.group(4);
743        }
744        // must create back reference here
745        HFileLink.create(conf, fs, splitDir, familyName, hri.getTable().getNameAsString(),
746          hri.getEncodedName(), linkedTable, linkedRegion, hfileName, true);
747        Path path =
748          new Path(splitDir, HFileLink.createHFileLinkName(linkedTable, linkedRegion, hfileName));
749        LOG.info("Created linkFile:" + path.toString() + " for child: " + hri.getEncodedName()
750          + ", parent: " + regionInfoForFs.getEncodedName());
751        return path;
752      } catch (IOException e) {
753        // if create HFileLink file failed, then just skip the error and create Reference file
754        LOG.error("Create link file for " + hfileName + " for child " + hri.getEncodedName()
755          + "failed, will create Reference file", e);
756      }
757    }
758    // A reference to the bottom half of the hsf store file.
759    Reference r =
760      top ? Reference.createTopReference(splitRow) : Reference.createBottomReference(splitRow);
761    return r.write(fs, p);
762  }
763
764  // ===========================================================================
765  // Merge Helpers
766  // ===========================================================================
767
768  Path getMergesDir(final RegionInfo hri) {
769    return new Path(getTableDir(), hri.getEncodedName());
770  }
771
772  /**
773   * Remove merged region
774   * @param mergedRegion {@link RegionInfo} n
775   */
776  public void cleanupMergedRegion(final RegionInfo mergedRegion) throws IOException {
777    Path regionDir = new Path(this.tableDir, mergedRegion.getEncodedName());
778    if (this.fs.exists(regionDir) && !this.fs.delete(regionDir, true)) {
779      throw new IOException("Failed delete of " + regionDir);
780    }
781  }
782
783  static boolean mkdirs(FileSystem fs, Configuration conf, Path dir) throws IOException {
784    if (
785      FSUtils.isDistributedFileSystem(fs)
786        || !conf.getBoolean(HConstants.ENABLE_DATA_FILE_UMASK, false)
787    ) {
788      return fs.mkdirs(dir);
789    }
790    FsPermission perms = CommonFSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
791    return fs.mkdirs(dir, perms);
792  }
793
794  /**
795   * Write out a merge reference under the given merges directory.
796   * @param mergingRegion {@link RegionInfo} for one of the regions being merged.
797   * @param familyName    Column Family Name
798   * @param f             File to create reference.
799   * @return Path to created reference.
800   * @throws IOException if the merge write fails.
801   */
802  public Path mergeStoreFile(RegionInfo mergingRegion, String familyName, HStoreFile f)
803    throws IOException {
804    Path referenceDir = new Path(getMergesDir(regionInfoForFs), familyName);
805    // A whole reference to the store file.
806    Reference r = Reference.createTopReference(mergingRegion.getStartKey());
807    // Add the referred-to regions name as a dot separated suffix.
808    // See REF_NAME_REGEX regex above. The referred-to regions name is
809    // up in the path of the passed in <code>f</code> -- parentdir is family,
810    // then the directory above is the region name.
811    String mergingRegionName = mergingRegion.getEncodedName();
812    // Write reference with same file id only with the other region name as
813    // suffix and into the new region location (under same family).
814    Path p = new Path(referenceDir, f.getPath().getName() + "." + mergingRegionName);
815    return r.write(fs, p);
816  }
817
818  /**
819   * Commit a merged region, making it ready for use. n
820   */
821  public void commitMergedRegion(List<Path> allMergedFiles, MasterProcedureEnv env)
822    throws IOException {
823    Path regionDir = getMergesDir(regionInfoForFs);
824    if (regionDir != null && fs.exists(regionDir)) {
825      // Write HRI to a file in case we need to recover hbase:meta
826      Path regionInfoFile = new Path(regionDir, REGION_INFO_FILE);
827      byte[] regionInfoContent = getRegionInfoFileContent(regionInfo);
828      writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
829      insertRegionFilesIntoStoreTracker(allMergedFiles, env, this);
830    }
831  }
832
833  // ===========================================================================
834  // Create/Open/Delete Helpers
835  // ===========================================================================
836
837  /**
838   * n * @return Content of the file we write out to the filesystem under a region n
839   */
840  private static byte[] getRegionInfoFileContent(final RegionInfo hri) throws IOException {
841    return RegionInfo.toDelimitedByteArray(hri);
842  }
843
844  /**
845   * Create a {@link RegionInfo} from the serialized version on-disk.
846   * @param fs        {@link FileSystem} that contains the Region Info file
847   * @param regionDir {@link Path} to the Region Directory that contains the Info file
848   * @return An {@link RegionInfo} instance gotten from the Region Info file.
849   * @throws IOException if an error occurred during file open/read operation.
850   */
851  public static RegionInfo loadRegionInfoFileContent(final FileSystem fs, final Path regionDir)
852    throws IOException {
853    FSDataInputStream in = fs.open(new Path(regionDir, REGION_INFO_FILE));
854    try {
855      return RegionInfo.parseFrom(in);
856    } finally {
857      in.close();
858    }
859  }
860
861  /**
862   * Write the .regioninfo file on-disk.
863   * <p/>
864   * Overwrites if exists already.
865   */
866  private static void writeRegionInfoFileContent(final Configuration conf, final FileSystem fs,
867    final Path regionInfoFile, final byte[] content) throws IOException {
868    // First check to get the permissions
869    FsPermission perms = CommonFSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
870    // Write the RegionInfo file content
871    try (FSDataOutputStream out = FSUtils.create(conf, fs, regionInfoFile, perms, null)) {
872      out.write(content);
873    }
874  }
875
876  /**
877   * Write out an info file under the stored region directory. Useful recovering mangled regions. If
878   * the regionInfo already exists on-disk, then we fast exit.
879   */
880  void checkRegionInfoOnFilesystem() throws IOException {
881    // Compose the content of the file so we can compare to length in filesystem. If not same,
882    // rewrite it (it may have been written in the old format using Writables instead of pb). The
883    // pb version is much shorter -- we write now w/o the toString version -- so checking length
884    // only should be sufficient. I don't want to read the file every time to check if it pb
885    // serialized.
886    byte[] content = getRegionInfoFileContent(regionInfoForFs);
887
888    // Verify if the region directory exists before opening a region. We need to do this since if
889    // the region directory doesn't exist we will re-create the region directory and a new HRI
890    // when HRegion.openHRegion() is called.
891    try {
892      FileStatus status = fs.getFileStatus(getRegionDir());
893    } catch (FileNotFoundException e) {
894      LOG.warn(getRegionDir() + " doesn't exist for region: " + regionInfoForFs.getEncodedName()
895        + " on table " + regionInfo.getTable());
896    }
897
898    try {
899      Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
900      FileStatus status = fs.getFileStatus(regionInfoFile);
901      if (status != null && status.getLen() == content.length) {
902        // Then assume the content good and move on.
903        // NOTE: that the length is not sufficient to define the the content matches.
904        return;
905      }
906
907      LOG.info("Rewriting .regioninfo file at: " + regionInfoFile);
908      if (!fs.delete(regionInfoFile, false)) {
909        throw new IOException("Unable to remove existing " + regionInfoFile);
910      }
911    } catch (FileNotFoundException e) {
912      LOG.warn(REGION_INFO_FILE + " file not found for region: " + regionInfoForFs.getEncodedName()
913        + " on table " + regionInfo.getTable());
914    }
915
916    // Write HRI to a file in case we need to recover hbase:meta
917    writeRegionInfoOnFilesystem(content, true);
918  }
919
920  /**
921   * Write out an info file under the region directory. Useful recovering mangled regions.
922   * @param useTempDir indicate whether or not using the region .tmp dir for a safer file creation.
923   */
924  private void writeRegionInfoOnFilesystem(boolean useTempDir) throws IOException {
925    byte[] content = getRegionInfoFileContent(regionInfoForFs);
926    writeRegionInfoOnFilesystem(content, useTempDir);
927  }
928
929  /**
930   * Write out an info file under the region directory. Useful recovering mangled regions.
931   * @param regionInfoContent serialized version of the {@link RegionInfo}
932   * @param useTempDir        indicate whether or not using the region .tmp dir for a safer file
933   *                          creation.
934   */
935  private void writeRegionInfoOnFilesystem(final byte[] regionInfoContent, final boolean useTempDir)
936    throws IOException {
937    Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
938    if (useTempDir) {
939      // Create in tmpDir and then move into place in case we crash after
940      // create but before close. If we don't successfully close the file,
941      // subsequent region reopens will fail the below because create is
942      // registered in NN.
943
944      // And then create the file
945      Path tmpPath = new Path(getTempDir(), REGION_INFO_FILE);
946
947      // If datanode crashes or if the RS goes down just before the close is called while trying to
948      // close the created regioninfo file in the .tmp directory then on next
949      // creation we will be getting AlreadyCreatedException.
950      // Hence delete and create the file if exists.
951      if (CommonFSUtils.isExists(fs, tmpPath)) {
952        CommonFSUtils.delete(fs, tmpPath, true);
953      }
954
955      // Write HRI to a file in case we need to recover hbase:meta
956      writeRegionInfoFileContent(conf, fs, tmpPath, regionInfoContent);
957
958      // Move the created file to the original path
959      if (fs.exists(tmpPath) && !rename(tmpPath, regionInfoFile)) {
960        throw new IOException("Unable to rename " + tmpPath + " to " + regionInfoFile);
961      }
962    } else {
963      // Write HRI to a file in case we need to recover hbase:meta
964      writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
965    }
966  }
967
968  /**
969   * Create a new Region on file-system.
970   * @param conf       the {@link Configuration} to use
971   * @param fs         {@link FileSystem} from which to add the region
972   * @param tableDir   {@link Path} to where the table is being stored
973   * @param regionInfo {@link RegionInfo} for region to be added
974   * @throws IOException if the region creation fails due to a FileSystem exception.
975   */
976  public static HRegionFileSystem createRegionOnFileSystem(final Configuration conf,
977    final FileSystem fs, final Path tableDir, final RegionInfo regionInfo) throws IOException {
978    HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
979
980    // We only create a .regioninfo and the region directory if this is the default region replica
981    if (regionInfo.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
982      Path regionDir = regionFs.getRegionDir();
983      if (fs.exists(regionDir)) {
984        LOG.warn("Trying to create a region that already exists on disk: " + regionDir);
985      } else {
986        // Create the region directory
987        if (!createDirOnFileSystem(fs, conf, regionDir)) {
988          LOG.warn("Unable to create the region directory: " + regionDir);
989          throw new IOException("Unable to create region directory: " + regionDir);
990        }
991      }
992
993      // Write HRI to a file in case we need to recover hbase:meta
994      regionFs.writeRegionInfoOnFilesystem(false);
995    } else {
996      if (LOG.isDebugEnabled())
997        LOG.debug("Skipping creation of .regioninfo file for " + regionInfo);
998    }
999    return regionFs;
1000  }
1001
1002  /**
1003   * Open Region from file-system.
1004   * @param conf       the {@link Configuration} to use
1005   * @param fs         {@link FileSystem} from which to add the region
1006   * @param tableDir   {@link Path} to where the table is being stored
1007   * @param regionInfo {@link RegionInfo} for region to be added
1008   * @param readOnly   True if you don't want to edit the region data
1009   * @throws IOException if the region creation fails due to a FileSystem exception.
1010   */
1011  public static HRegionFileSystem openRegionFromFileSystem(final Configuration conf,
1012    final FileSystem fs, final Path tableDir, final RegionInfo regionInfo, boolean readOnly)
1013    throws IOException {
1014    HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
1015    Path regionDir = regionFs.getRegionDir();
1016
1017    if (!fs.exists(regionDir)) {
1018      LOG.warn("Trying to open a region that do not exists on disk: " + regionDir);
1019      throw new IOException("The specified region do not exists on disk: " + regionDir);
1020    }
1021
1022    if (!readOnly) {
1023      // Cleanup temporary directories
1024      regionFs.cleanupTempDir();
1025
1026      // If it doesn't exists, Write HRI to a file, in case we need to recover hbase:meta
1027      // Only create HRI if we are the default replica
1028      if (regionInfo.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
1029        regionFs.checkRegionInfoOnFilesystem();
1030      } else {
1031        if (LOG.isDebugEnabled()) {
1032          LOG.debug("Skipping creation of .regioninfo file for " + regionInfo);
1033        }
1034      }
1035    }
1036
1037    return regionFs;
1038  }
1039
1040  /**
1041   * Remove the region from the table directory, archiving the region's hfiles.
1042   * @param conf       the {@link Configuration} to use
1043   * @param fs         {@link FileSystem} from which to remove the region
1044   * @param tableDir   {@link Path} to where the table is being stored
1045   * @param regionInfo {@link RegionInfo} for region to be deleted
1046   * @throws IOException if the request cannot be completed
1047   */
1048  public static void deleteRegionFromFileSystem(final Configuration conf, final FileSystem fs,
1049    final Path tableDir, final RegionInfo regionInfo) throws IOException {
1050    HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
1051    Path regionDir = regionFs.getRegionDir();
1052
1053    if (!fs.exists(regionDir)) {
1054      LOG.warn("Trying to delete a region that do not exists on disk: " + regionDir);
1055      return;
1056    }
1057
1058    if (LOG.isDebugEnabled()) {
1059      LOG.debug("DELETING region " + regionDir);
1060    }
1061
1062    // Archive region
1063    Path rootDir = CommonFSUtils.getRootDir(conf);
1064    HFileArchiver.archiveRegion(fs, rootDir, tableDir, regionDir);
1065
1066    // Delete empty region dir
1067    if (!fs.delete(regionDir, true)) {
1068      LOG.warn("Failed delete of " + regionDir);
1069    }
1070  }
1071
1072  /**
1073   * Creates a directory. Assumes the user has already checked for this directory existence. n
1074   * * @return the result of fs.mkdirs(). In case underlying fs throws an IOException, it checks
1075   * whether the directory exists or not, and returns true if it exists. n
1076   */
1077  boolean createDir(Path dir) throws IOException {
1078    int i = 0;
1079    IOException lastIOE = null;
1080    do {
1081      try {
1082        return mkdirs(fs, conf, dir);
1083      } catch (IOException ioe) {
1084        lastIOE = ioe;
1085        if (fs.exists(dir)) return true; // directory is present
1086        try {
1087          sleepBeforeRetry("Create Directory", i + 1);
1088        } catch (InterruptedException e) {
1089          throw (InterruptedIOException) new InterruptedIOException().initCause(e);
1090        }
1091      }
1092    } while (++i <= hdfsClientRetriesNumber);
1093    throw new IOException("Exception in createDir", lastIOE);
1094  }
1095
1096  /**
1097   * Renames a directory. Assumes the user has already checked for this directory existence. nn
1098   * * @return true if rename is successful. n
1099   */
1100  boolean rename(Path srcpath, Path dstPath) throws IOException {
1101    IOException lastIOE = null;
1102    int i = 0;
1103    do {
1104      try {
1105        return fs.rename(srcpath, dstPath);
1106      } catch (IOException ioe) {
1107        lastIOE = ioe;
1108        if (!fs.exists(srcpath) && fs.exists(dstPath)) return true; // successful move
1109        // dir is not there, retry after some time.
1110        try {
1111          sleepBeforeRetry("Rename Directory", i + 1);
1112        } catch (InterruptedException e) {
1113          throw (InterruptedIOException) new InterruptedIOException().initCause(e);
1114        }
1115      }
1116    } while (++i <= hdfsClientRetriesNumber);
1117
1118    throw new IOException("Exception in rename", lastIOE);
1119  }
1120
1121  /**
1122   * Deletes a directory. Assumes the user has already checked for this directory existence. n
1123   * * @return true if the directory is deleted. n
1124   */
1125  boolean deleteDir(Path dir) throws IOException {
1126    IOException lastIOE = null;
1127    int i = 0;
1128    do {
1129      try {
1130        return fs.delete(dir, true);
1131      } catch (IOException ioe) {
1132        lastIOE = ioe;
1133        if (!fs.exists(dir)) return true;
1134        // dir is there, retry deleting after some time.
1135        try {
1136          sleepBeforeRetry("Delete Directory", i + 1);
1137        } catch (InterruptedException e) {
1138          throw (InterruptedIOException) new InterruptedIOException().initCause(e);
1139        }
1140      }
1141    } while (++i <= hdfsClientRetriesNumber);
1142
1143    throw new IOException("Exception in DeleteDir", lastIOE);
1144  }
1145
1146  /**
1147   * sleeping logic; handles the interrupt exception.
1148   */
1149  private void sleepBeforeRetry(String msg, int sleepMultiplier) throws InterruptedException {
1150    sleepBeforeRetry(msg, sleepMultiplier, baseSleepBeforeRetries, hdfsClientRetriesNumber);
1151  }
1152
1153  /**
1154   * Creates a directory for a filesystem and configuration object. Assumes the user has already
1155   * checked for this directory existence. nnn * @return the result of fs.mkdirs(). In case
1156   * underlying fs throws an IOException, it checks whether the directory exists or not, and returns
1157   * true if it exists. n
1158   */
1159  private static boolean createDirOnFileSystem(FileSystem fs, Configuration conf, Path dir)
1160    throws IOException {
1161    int i = 0;
1162    IOException lastIOE = null;
1163    int hdfsClientRetriesNumber =
1164      conf.getInt("hdfs.client.retries.number", DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
1165    int baseSleepBeforeRetries =
1166      conf.getInt("hdfs.client.sleep.before.retries", DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
1167    do {
1168      try {
1169        return fs.mkdirs(dir);
1170      } catch (IOException ioe) {
1171        lastIOE = ioe;
1172        if (fs.exists(dir)) return true; // directory is present
1173        try {
1174          sleepBeforeRetry("Create Directory", i + 1, baseSleepBeforeRetries,
1175            hdfsClientRetriesNumber);
1176        } catch (InterruptedException e) {
1177          throw (InterruptedIOException) new InterruptedIOException().initCause(e);
1178        }
1179      }
1180    } while (++i <= hdfsClientRetriesNumber);
1181
1182    throw new IOException("Exception in createDir", lastIOE);
1183  }
1184
1185  /**
1186   * sleeping logic for static methods; handles the interrupt exception. Keeping a static version
1187   * for this to avoid re-looking for the integer values.
1188   */
1189  private static void sleepBeforeRetry(String msg, int sleepMultiplier, int baseSleepBeforeRetries,
1190    int hdfsClientRetriesNumber) throws InterruptedException {
1191    if (sleepMultiplier > hdfsClientRetriesNumber) {
1192      if (LOG.isDebugEnabled()) {
1193        LOG.debug(msg + ", retries exhausted");
1194      }
1195      return;
1196    }
1197    if (LOG.isDebugEnabled()) {
1198      LOG.debug(msg + ", sleeping " + baseSleepBeforeRetries + " times " + sleepMultiplier);
1199    }
1200    Thread.sleep((long) baseSleepBeforeRetries * sleepMultiplier);
1201  }
1202}