001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.io.HFileLink.LINK_NAME_PATTERN;
021
022import edu.umd.cs.findbugs.annotations.Nullable;
023import java.io.FileNotFoundException;
024import java.io.IOException;
025import java.io.InterruptedIOException;
026import java.util.ArrayList;
027import java.util.Collection;
028import java.util.HashMap;
029import java.util.List;
030import java.util.Map;
031import java.util.Objects;
032import java.util.Optional;
033import java.util.UUID;
034import java.util.regex.Matcher;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.fs.FSDataInputStream;
037import org.apache.hadoop.fs.FSDataOutputStream;
038import org.apache.hadoop.fs.FileStatus;
039import org.apache.hadoop.fs.FileSystem;
040import org.apache.hadoop.fs.FileUtil;
041import org.apache.hadoop.fs.LocatedFileStatus;
042import org.apache.hadoop.fs.Path;
043import org.apache.hadoop.fs.permission.FsPermission;
044import org.apache.hadoop.hbase.Cell;
045import org.apache.hadoop.hbase.HConstants;
046import org.apache.hadoop.hbase.PrivateCellUtil;
047import org.apache.hadoop.hbase.TableName;
048import org.apache.hadoop.hbase.backup.HFileArchiver;
049import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
050import org.apache.hadoop.hbase.client.RegionInfo;
051import org.apache.hadoop.hbase.client.TableDescriptor;
052import org.apache.hadoop.hbase.fs.HFileSystem;
053import org.apache.hadoop.hbase.io.HFileLink;
054import org.apache.hadoop.hbase.io.Reference;
055import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
056import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
057import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
058import org.apache.hadoop.hbase.util.Bytes;
059import org.apache.hadoop.hbase.util.CommonFSUtils;
060import org.apache.hadoop.hbase.util.FSUtils;
061import org.apache.hadoop.hbase.util.Pair;
062import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
063import org.apache.yetus.audience.InterfaceAudience;
064import org.slf4j.Logger;
065import org.slf4j.LoggerFactory;
066
067import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
068
069/**
070 * View to an on-disk Region. Provides the set of methods necessary to interact with the on-disk
071 * region data.
072 */
073@InterfaceAudience.Private
074public class HRegionFileSystem {
075  private static final Logger LOG = LoggerFactory.getLogger(HRegionFileSystem.class);
076
077  /** Name of the region info file that resides just under the region directory. */
078  public final static String REGION_INFO_FILE = ".regioninfo";
079
080  /** Temporary subdirectory of the region directory used for merges. */
081  public static final String REGION_MERGES_DIR = ".merges";
082
083  /** Temporary subdirectory of the region directory used for splits. */
084  public static final String REGION_SPLITS_DIR = ".splits";
085
086  /** Temporary subdirectory of the region directory used for compaction output. */
087  static final String REGION_TEMP_DIR = ".tmp";
088
089  private final RegionInfo regionInfo;
090  // regionInfo for interacting with FS (getting encodedName, etc)
091  final RegionInfo regionInfoForFs;
092  final Configuration conf;
093  private final Path tableDir;
094  final FileSystem fs;
095  private final Path regionDir;
096
097  /**
098   * In order to handle NN connectivity hiccups, one need to retry non-idempotent operation at the
099   * client level.
100   */
101  private final int hdfsClientRetriesNumber;
102  private final int baseSleepBeforeRetries;
103  private static final int DEFAULT_HDFS_CLIENT_RETRIES_NUMBER = 10;
104  private static final int DEFAULT_BASE_SLEEP_BEFORE_RETRIES = 1000;
105
106  /**
107   * Create a view to the on-disk region
108   * @param conf       the {@link Configuration} to use
109   * @param fs         {@link FileSystem} that contains the region
110   * @param tableDir   {@link Path} to where the table is being stored
111   * @param regionInfo {@link RegionInfo} for region
112   */
113  HRegionFileSystem(final Configuration conf, final FileSystem fs, final Path tableDir,
114    final RegionInfo regionInfo) {
115    this.fs = fs;
116    this.conf = conf;
117    this.tableDir = Objects.requireNonNull(tableDir, "tableDir is null");
118    this.regionInfo = Objects.requireNonNull(regionInfo, "regionInfo is null");
119    this.regionInfoForFs = ServerRegionReplicaUtil.getRegionInfoForFs(regionInfo);
120    this.regionDir = FSUtils.getRegionDirFromTableDir(tableDir, regionInfo);
121    this.hdfsClientRetriesNumber =
122      conf.getInt("hdfs.client.retries.number", DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
123    this.baseSleepBeforeRetries =
124      conf.getInt("hdfs.client.sleep.before.retries", DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
125  }
126
127  /** Returns the underlying {@link FileSystem} */
128  public FileSystem getFileSystem() {
129    return this.fs;
130  }
131
132  /** Returns the {@link RegionInfo} that describe this on-disk region view */
133  public RegionInfo getRegionInfo() {
134    return this.regionInfo;
135  }
136
137  public RegionInfo getRegionInfoForFS() {
138    return this.regionInfoForFs;
139  }
140
141  /** Returns {@link Path} to the region's root directory. */
142  public Path getTableDir() {
143    return this.tableDir;
144  }
145
146  /** Returns {@link Path} to the region directory. */
147  public Path getRegionDir() {
148    return regionDir;
149  }
150
151  // ===========================================================================
152  // Temp Helpers
153  // ===========================================================================
154  /** Returns {@link Path} to the region's temp directory, used for file creations */
155  public Path getTempDir() {
156    return new Path(getRegionDir(), REGION_TEMP_DIR);
157  }
158
159  /**
160   * Clean up any temp detritus that may have been left around from previous operation attempts.
161   */
162  void cleanupTempDir() throws IOException {
163    deleteDir(getTempDir());
164  }
165
166  // ===========================================================================
167  // Store/StoreFile Helpers
168  // ===========================================================================
169  /**
170   * Returns the directory path of the specified family
171   * @param familyName Column Family Name
172   * @return {@link Path} to the directory of the specified family
173   */
174  public Path getStoreDir(final String familyName) {
175    return new Path(this.getRegionDir(), familyName);
176  }
177
178  /**
179   * @param tabledir {@link Path} to where the table is being stored
180   * @param hri      {@link RegionInfo} for the region.
181   * @param family   {@link ColumnFamilyDescriptor} describing the column family
182   * @return Path to family/Store home directory.
183   */
184  public static Path getStoreHomedir(final Path tabledir, final RegionInfo hri,
185    final byte[] family) {
186    return getStoreHomedir(tabledir, hri.getEncodedName(), family);
187  }
188
189  /**
190   * @param tabledir    {@link Path} to where the table is being stored
191   * @param encodedName Encoded region name.
192   * @param family      {@link ColumnFamilyDescriptor} describing the column family
193   * @return Path to family/Store home directory.
194   */
195  public static Path getStoreHomedir(final Path tabledir, final String encodedName,
196    final byte[] family) {
197    return new Path(tabledir, new Path(encodedName, Bytes.toString(family)));
198  }
199
200  /**
201   * Create the store directory for the specified family name
202   * @param familyName Column Family Name
203   * @return {@link Path} to the directory of the specified family
204   * @throws IOException if the directory creation fails.
205   */
206  Path createStoreDir(final String familyName) throws IOException {
207    Path storeDir = getStoreDir(familyName);
208    if (!fs.exists(storeDir) && !createDir(storeDir))
209      throw new IOException("Failed creating " + storeDir);
210    return storeDir;
211  }
212
213  /**
214   * Set the directory of CF to the specified storage policy. <br>
215   * <i>"LAZY_PERSIST"</i>, <i>"ALL_SSD"</i>, <i>"ONE_SSD"</i>, <i>"HOT"</i>, <i>"WARM"</i>,
216   * <i>"COLD"</i> <br>
217   * <br>
218   * See {@link org.apache.hadoop.hdfs.protocol.HdfsConstants} for more details.
219   * @param familyName The name of column family.
220   * @param policyName The name of the storage policy: 'HOT', 'COLD', etc. See hadoop 2.6+
221   *                   org.apache.hadoop.hdfs.protocol.HdfsConstants for possible list e.g 'COLD',
222   *                   'WARM', 'HOT', 'ONE_SSD', 'ALL_SSD', 'LAZY_PERSIST'.
223   */
224  public void setStoragePolicy(String familyName, String policyName) {
225    CommonFSUtils.setStoragePolicy(this.fs, getStoreDir(familyName), policyName);
226  }
227
228  /**
229   * Set storage policy for a whole region. <br>
230   * <i>"LAZY_PERSIST"</i>, <i>"ALL_SSD"</i>, <i>"ONE_SSD"</i>, <i>"HOT"</i>, <i>"WARM"</i>,
231   * <i>"COLD"</i> <br>
232   * <br>
233   * See {@link org.apache.hadoop.hdfs.protocol.HdfsConstants} for more details.
234   * @param policyName The name of the storage policy: 'HOT', 'COLD', etc. See hadoop 2.6+
235   *                   org.apache.hadoop.hdfs.protocol.HdfsConstants for possible list e.g 'COLD',
236   *                   'WARM', 'HOT', 'ONE_SSD', 'ALL_SSD', 'LAZY_PERSIST'.
237   */
238  public void setStoragePolicy(String policyName) {
239    CommonFSUtils.setStoragePolicy(this.fs, getRegionDir(), policyName);
240  }
241
242  /**
243   * Get the storage policy of the directory of CF.
244   * @param familyName The name of column family.
245   * @return Storage policy name, or {@code null} if not using {@link HFileSystem} or exception
246   *         thrown when trying to get policy
247   */
248  @Nullable
249  public String getStoragePolicyName(String familyName) {
250    if (this.fs instanceof HFileSystem) {
251      Path storeDir = getStoreDir(familyName);
252      return ((HFileSystem) this.fs).getStoragePolicyName(storeDir);
253    }
254
255    return null;
256  }
257
258  /**
259   * Returns the store files available for the family. This methods performs the filtering based on
260   * the valid store files.
261   * @param familyName Column Family Name
262   * @return a set of {@link StoreFileInfo} for the specified family.
263   */
264  public List<StoreFileInfo> getStoreFiles(final String familyName) throws IOException {
265    return getStoreFiles(familyName, true);
266  }
267
268  /**
269   * Returns the store files available for the family. This methods performs the filtering based on
270   * the valid store files.
271   * @param familyName Column Family Name
272   * @return a set of {@link StoreFileInfo} for the specified family.
273   */
274  public List<StoreFileInfo> getStoreFiles(final String familyName, final boolean validate)
275    throws IOException {
276    Path familyDir = getStoreDir(familyName);
277    FileStatus[] files = CommonFSUtils.listStatus(this.fs, familyDir);
278    if (files == null) {
279      if (LOG.isTraceEnabled()) {
280        LOG.trace("No StoreFiles for: " + familyDir);
281      }
282      return null;
283    }
284
285    ArrayList<StoreFileInfo> storeFiles = new ArrayList<>(files.length);
286    for (FileStatus status : files) {
287      if (validate && !StoreFileInfo.isValid(status)) {
288        // recovered.hfiles directory is expected inside CF path when hbase.wal.split.to.hfile to
289        // true, refer HBASE-23740
290        if (!HConstants.RECOVERED_HFILES_DIR.equals(status.getPath().getName())) {
291          LOG.warn("Invalid StoreFile: {}", status.getPath());
292        }
293        continue;
294      }
295      StoreFileInfo info = ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, regionInfo,
296        regionInfoForFs, familyName, status.getPath());
297      storeFiles.add(info);
298
299    }
300    return storeFiles;
301  }
302
303  /**
304   * Returns the store files' LocatedFileStatus which available for the family. This methods
305   * performs the filtering based on the valid store files.
306   * @param familyName Column Family Name
307   * @return a list of store files' LocatedFileStatus for the specified family.
308   */
309  public static List<LocatedFileStatus> getStoreFilesLocatedStatus(final HRegionFileSystem regionfs,
310    final String familyName, final boolean validate) throws IOException {
311    Path familyDir = regionfs.getStoreDir(familyName);
312    List<LocatedFileStatus> locatedFileStatuses =
313      CommonFSUtils.listLocatedStatus(regionfs.getFileSystem(), familyDir);
314    if (locatedFileStatuses == null) {
315      if (LOG.isTraceEnabled()) {
316        LOG.trace("No StoreFiles for: " + familyDir);
317      }
318      return null;
319    }
320
321    List<LocatedFileStatus> validStoreFiles = Lists.newArrayList();
322    for (LocatedFileStatus status : locatedFileStatuses) {
323      if (validate && !StoreFileInfo.isValid(status)) {
324        // recovered.hfiles directory is expected inside CF path when hbase.wal.split.to.hfile to
325        // true, refer HBASE-23740
326        if (!HConstants.RECOVERED_HFILES_DIR.equals(status.getPath().getName())) {
327          LOG.warn("Invalid StoreFile: {}", status.getPath());
328        }
329      } else {
330        validStoreFiles.add(status);
331      }
332    }
333    return validStoreFiles;
334  }
335
336  /**
337   * Return Qualified Path of the specified family/file
338   * @param familyName Column Family Name
339   * @param fileName   File Name
340   * @return The qualified Path for the specified family/file
341   */
342  Path getStoreFilePath(final String familyName, final String fileName) {
343    Path familyDir = getStoreDir(familyName);
344    return new Path(familyDir, fileName).makeQualified(fs.getUri(), fs.getWorkingDirectory());
345  }
346
347  /**
348   * Return the store file information of the specified family/file.
349   * @param familyName Column Family Name
350   * @param fileName   File Name
351   * @return The {@link StoreFileInfo} for the specified family/file
352   */
353  StoreFileInfo getStoreFileInfo(final String familyName, final String fileName)
354    throws IOException {
355    Path familyDir = getStoreDir(familyName);
356    return ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, regionInfo, regionInfoForFs,
357      familyName, new Path(familyDir, fileName));
358  }
359
360  /**
361   * Returns true if the specified family has reference files
362   * @param familyName Column Family Name
363   * @return true if family contains reference files
364   */
365  public boolean hasReferences(final String familyName) throws IOException {
366    Path storeDir = getStoreDir(familyName);
367    FileStatus[] files = CommonFSUtils.listStatus(fs, storeDir);
368    if (files != null) {
369      for (FileStatus stat : files) {
370        if (stat.isDirectory()) {
371          continue;
372        }
373        if (StoreFileInfo.isReference(stat.getPath())) {
374          LOG.trace("Reference {}", stat.getPath());
375          return true;
376        }
377      }
378    }
379    return false;
380  }
381
382  /**
383   * Check whether region has Reference file
384   * @param htd table desciptor of the region
385   * @return true if region has reference file
386   */
387  public boolean hasReferences(final TableDescriptor htd) throws IOException {
388    for (ColumnFamilyDescriptor family : htd.getColumnFamilies()) {
389      if (hasReferences(family.getNameAsString())) {
390        return true;
391      }
392    }
393    return false;
394  }
395
396  /** Returns the set of families present on disk n */
397  public Collection<String> getFamilies() throws IOException {
398    FileStatus[] fds =
399      CommonFSUtils.listStatus(fs, getRegionDir(), new FSUtils.FamilyDirFilter(fs));
400    if (fds == null) return null;
401
402    ArrayList<String> families = new ArrayList<>(fds.length);
403    for (FileStatus status : fds) {
404      families.add(status.getPath().getName());
405    }
406
407    return families;
408  }
409
410  /**
411   * Remove the region family from disk, archiving the store files.
412   * @param familyName Column Family Name
413   * @throws IOException if an error occours during the archiving
414   */
415  public void deleteFamily(final String familyName) throws IOException {
416    // archive family store files
417    HFileArchiver.archiveFamily(fs, conf, regionInfoForFs, tableDir, Bytes.toBytes(familyName));
418
419    // delete the family folder
420    Path familyDir = getStoreDir(familyName);
421    if (fs.exists(familyDir) && !deleteDir(familyDir))
422      throw new IOException("Could not delete family " + familyName + " from FileSystem for region "
423        + regionInfoForFs.getRegionNameAsString() + "(" + regionInfoForFs.getEncodedName() + ")");
424  }
425
426  /**
427   * Generate a unique file name, used by createTempName() and commitStoreFile()
428   * @param suffix extra information to append to the generated name
429   * @return Unique file name
430   */
431  private static String generateUniqueName(final String suffix) {
432    String name = UUID.randomUUID().toString().replaceAll("-", "");
433    if (suffix != null) name += suffix;
434    return name;
435  }
436
437  /**
438   * Generate a unique temporary Path. Used in conjuction with commitStoreFile() to get a safer file
439   * creation. <code>
440   * Path file = fs.createTempName();
441   * ...StoreFile.Writer(file)...
442   * fs.commitStoreFile("family", file);
443   * </code>
444   * @return Unique {@link Path} of the temporary file
445   */
446  public Path createTempName() {
447    return createTempName(null);
448  }
449
450  /**
451   * Generate a unique temporary Path. Used in conjuction with commitStoreFile() to get a safer file
452   * creation. <code>
453   * Path file = fs.createTempName();
454   * ...StoreFile.Writer(file)...
455   * fs.commitStoreFile("family", file);
456   * </code>
457   * @param suffix extra information to append to the generated name
458   * @return Unique {@link Path} of the temporary file
459   */
460  public Path createTempName(final String suffix) {
461    return new Path(getTempDir(), generateUniqueName(suffix));
462  }
463
464  /**
465   * Move the file from a build/temp location to the main family store directory.
466   * @param familyName Family that will gain the file
467   * @param buildPath  {@link Path} to the file to commit.
468   * @return The new {@link Path} of the committed file
469   */
470  public Path commitStoreFile(final String familyName, final Path buildPath) throws IOException {
471    Path dstPath = preCommitStoreFile(familyName, buildPath, -1, false);
472    return commitStoreFile(buildPath, dstPath);
473  }
474
475  /**
476   * Generate the filename in the main family store directory for moving the file from a build/temp
477   * location.
478   * @param familyName      Family that will gain the file
479   * @param buildPath       {@link Path} to the file to commit.
480   * @param seqNum          Sequence Number to append to the file name (less then 0 if no sequence
481   *                        number)
482   * @param generateNewName False if you want to keep the buildPath name
483   * @return The new {@link Path} of the to be committed file
484   */
485  private Path preCommitStoreFile(final String familyName, final Path buildPath, final long seqNum,
486    final boolean generateNewName) throws IOException {
487    Path storeDir = getStoreDir(familyName);
488    if (!fs.exists(storeDir) && !createDir(storeDir))
489      throw new IOException("Failed creating " + storeDir);
490
491    String name = buildPath.getName();
492    if (generateNewName) {
493      name = generateUniqueName((seqNum < 0) ? null : "_SeqId_" + seqNum + "_");
494    }
495    Path dstPath = new Path(storeDir, name);
496    if (!fs.exists(buildPath)) {
497      throw new FileNotFoundException(buildPath.toString());
498    }
499    if (LOG.isDebugEnabled()) {
500      LOG.debug("Committing " + buildPath + " as " + dstPath);
501    }
502    return dstPath;
503  }
504
505  /*
506   * Moves file from staging dir to region dir
507   * @param buildPath {@link Path} to the file to commit.
508   * @param dstPath {@link Path} to the file under region dir
509   * @return The {@link Path} of the committed file
510   */
511  Path commitStoreFile(final Path buildPath, Path dstPath) throws IOException {
512    // rename is not necessary in case of direct-insert stores
513    if (buildPath.equals(dstPath)) {
514      return dstPath;
515    }
516    // buildPath exists, therefore not doing an exists() check.
517    if (!rename(buildPath, dstPath)) {
518      throw new IOException("Failed rename of " + buildPath + " to " + dstPath);
519    }
520    return dstPath;
521  }
522
523  /**
524   * Archives the specified store file from the specified family.
525   * @param familyName Family that contains the store files
526   * @param filePath   {@link Path} to the store file to remove
527   * @throws IOException if the archiving fails
528   */
529  public void removeStoreFile(final String familyName, final Path filePath) throws IOException {
530    HFileArchiver.archiveStoreFile(this.conf, this.fs, this.regionInfoForFs, this.tableDir,
531      Bytes.toBytes(familyName), filePath);
532  }
533
534  /**
535   * Closes and archives the specified store files from the specified family.
536   * @param familyName Family that contains the store files
537   * @param storeFiles set of store files to remove
538   * @throws IOException if the archiving fails
539   */
540  public void removeStoreFiles(String familyName, Collection<HStoreFile> storeFiles)
541    throws IOException {
542    HFileArchiver.archiveStoreFiles(this.conf, this.fs, this.regionInfoForFs, this.tableDir,
543      Bytes.toBytes(familyName), storeFiles);
544  }
545
546  /**
547   * Bulk load: Add a specified store file to the specified family. If the source file is on the
548   * same different file-system is moved from the source location to the destination location,
549   * otherwise is copied over.
550   * @param familyName Family that will gain the file
551   * @param srcPath    {@link Path} to the file to import
552   * @param seqNum     Bulk Load sequence number
553   * @return The destination {@link Path} of the bulk loaded file
554   */
555  Pair<Path, Path> bulkLoadStoreFile(final String familyName, Path srcPath, long seqNum)
556    throws IOException {
557    // Copy the file if it's on another filesystem
558    FileSystem srcFs = srcPath.getFileSystem(conf);
559    srcPath = srcFs.resolvePath(srcPath);
560    FileSystem realSrcFs = srcPath.getFileSystem(conf);
561    FileSystem desFs = fs instanceof HFileSystem ? ((HFileSystem) fs).getBackingFs() : fs;
562
563    // We can't compare FileSystem instances as equals() includes UGI instance
564    // as part of the comparison and won't work when doing SecureBulkLoad
565    // TODO deal with viewFS
566    if (!FSUtils.isSameHdfs(conf, realSrcFs, desFs)) {
567      LOG.info("Bulk-load file " + srcPath + " is on different filesystem than "
568        + "the destination store. Copying file over to destination filesystem.");
569      Path tmpPath = createTempName();
570      FileUtil.copy(realSrcFs, srcPath, fs, tmpPath, false, conf);
571      LOG.info("Copied " + srcPath + " to temporary path on destination filesystem: " + tmpPath);
572      srcPath = tmpPath;
573    }
574
575    return new Pair<>(srcPath, preCommitStoreFile(familyName, srcPath, seqNum, true));
576  }
577
578  // ===========================================================================
579  // Splits Helpers
580  // ===========================================================================
581
582  public Path getSplitsDir(final RegionInfo hri) {
583    return new Path(getTableDir(), hri.getEncodedName());
584  }
585
586  /**
587   * Remove daughter region
588   * @param regionInfo daughter {@link RegionInfo}
589   */
590  void cleanupDaughterRegion(final RegionInfo regionInfo) throws IOException {
591    Path regionDir = new Path(this.tableDir, regionInfo.getEncodedName());
592    if (this.fs.exists(regionDir) && !deleteDir(regionDir)) {
593      throw new IOException("Failed delete of " + regionDir);
594    }
595  }
596
597  /**
598   * Commit a daughter region, moving it from the split temporary directory to the proper location
599   * in the filesystem.
600   * @param regionInfo daughter {@link org.apache.hadoop.hbase.client.RegionInfo}
601   */
602  public Path commitDaughterRegion(final RegionInfo regionInfo, List<Path> allRegionFiles,
603    MasterProcedureEnv env) throws IOException {
604    Path regionDir = this.getSplitsDir(regionInfo);
605    if (fs.exists(regionDir)) {
606      // Write HRI to a file in case we need to recover hbase:meta
607      Path regionInfoFile = new Path(regionDir, REGION_INFO_FILE);
608      byte[] regionInfoContent = getRegionInfoFileContent(regionInfo);
609      writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
610      HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
611        env.getMasterConfiguration(), fs, getTableDir(), regionInfo, false);
612      insertRegionFilesIntoStoreTracker(allRegionFiles, env, regionFs);
613    }
614    return regionDir;
615  }
616
617  private void insertRegionFilesIntoStoreTracker(List<Path> allFiles, MasterProcedureEnv env,
618    HRegionFileSystem regionFs) throws IOException {
619    TableDescriptor tblDesc =
620      env.getMasterServices().getTableDescriptors().get(regionInfo.getTable());
621    // we need to map trackers per store
622    Map<String, StoreFileTracker> trackerMap = new HashMap<>();
623    // we need to map store files per store
624    Map<String, List<StoreFileInfo>> fileInfoMap = new HashMap<>();
625    for (Path file : allFiles) {
626      String familyName = file.getParent().getName();
627      trackerMap.computeIfAbsent(familyName, t -> StoreFileTrackerFactory.create(conf, tblDesc,
628        tblDesc.getColumnFamily(Bytes.toBytes(familyName)), regionFs));
629      fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>());
630      List<StoreFileInfo> infos = fileInfoMap.get(familyName);
631      infos.add(new StoreFileInfo(conf, fs, file, true));
632    }
633    for (Map.Entry<String, StoreFileTracker> entry : trackerMap.entrySet()) {
634      entry.getValue().add(fileInfoMap.get(entry.getKey()));
635    }
636  }
637
638  /**
639   * Creates region split daughter directories under the table dir. If the daughter regions already
640   * exist, for example, in the case of a recovery from a previous failed split procedure, this
641   * method deletes the given region dir recursively, then recreates it again.
642   */
643  public void createSplitsDir(RegionInfo daughterA, RegionInfo daughterB) throws IOException {
644    Path daughterADir = getSplitsDir(daughterA);
645    if (fs.exists(daughterADir) && !deleteDir(daughterADir)) {
646      throw new IOException("Failed deletion of " + daughterADir + " before creating them again.");
647
648    }
649    if (!createDir(daughterADir)) {
650      throw new IOException("Failed create of " + daughterADir);
651    }
652    Path daughterBDir = getSplitsDir(daughterB);
653    if (fs.exists(daughterBDir) && !deleteDir(daughterBDir)) {
654      throw new IOException("Failed deletion of " + daughterBDir + " before creating them again.");
655
656    }
657    if (!createDir(daughterBDir)) {
658      throw new IOException("Failed create of " + daughterBDir);
659    }
660  }
661
662  /**
663   * Write out a split reference. Package local so it doesnt leak out of regionserver.
664   * @param hri         {@link RegionInfo} of the destination
665   * @param familyName  Column Family Name
666   * @param f           File to split.
667   * @param splitRow    Split Row
668   * @param top         True if we are referring to the top half of the hfile.
669   * @param splitPolicy A split policy instance; be careful! May not be full populated; e.g. if this
670   *                    method is invoked on the Master side, then the RegionSplitPolicy will NOT
671   *                    have a reference to a Region.
672   * @return Path to created reference.
673   */
674  public Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, byte[] splitRow,
675    boolean top, RegionSplitPolicy splitPolicy) throws IOException {
676    Path splitDir = new Path(getSplitsDir(hri), familyName);
677    // Add the referred-to regions name as a dot separated suffix.
678    // See REF_NAME_REGEX regex above. The referred-to regions name is
679    // up in the path of the passed in <code>f</code> -- parentdir is family,
680    // then the directory above is the region name.
681    String parentRegionName = regionInfoForFs.getEncodedName();
682    // Write reference with same file id only with the other region name as
683    // suffix and into the new region location (under same family).
684    Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
685    if (fs.exists(p)) {
686      LOG.warn("Found an already existing split file for {}. Assuming this is a recovery.", p);
687      return p;
688    }
689    boolean createLinkFile = false;
690    if (splitPolicy == null || !splitPolicy.skipStoreFileRangeCheck(familyName)) {
691      // Check whether the split row lies in the range of the store file
692      // If it is outside the range, return directly.
693      f.initReader();
694      try {
695        Cell splitKey = PrivateCellUtil.createFirstOnRow(splitRow);
696        Optional<Cell> lastKey = f.getLastKey();
697        Optional<Cell> firstKey = f.getFirstKey();
698        if (top) {
699          // check if larger than last key.
700          // If lastKey is null means storefile is empty.
701          if (!lastKey.isPresent()) {
702            return null;
703          }
704          if (f.getComparator().compare(splitKey, lastKey.get()) > 0) {
705            return null;
706          }
707          if (firstKey.isPresent() && f.getComparator().compare(splitKey, firstKey.get()) <= 0) {
708            LOG.debug("Will create HFileLink file for {}, top=true", f.getPath());
709            createLinkFile = true;
710          }
711        } else {
712          // check if smaller than first key
713          // If firstKey is null means storefile is empty.
714          if (!firstKey.isPresent()) {
715            return null;
716          }
717          if (f.getComparator().compare(splitKey, firstKey.get()) < 0) {
718            return null;
719          }
720          if (lastKey.isPresent() && f.getComparator().compare(splitKey, lastKey.get()) >= 0) {
721            LOG.debug("Will create HFileLink file for {}, top=false", f.getPath());
722            createLinkFile = true;
723          }
724        }
725      } finally {
726        f.closeStoreFile(f.getCacheConf() != null ? f.getCacheConf().shouldEvictOnClose() : true);
727      }
728    }
729    if (createLinkFile) {
730      // create HFileLink file instead of Reference file for child
731      String hfileName = f.getPath().getName();
732      TableName linkedTable = regionInfoForFs.getTable();
733      String linkedRegion = regionInfoForFs.getEncodedName();
734      try {
735        if (HFileLink.isHFileLink(hfileName)) {
736          Matcher m = LINK_NAME_PATTERN.matcher(hfileName);
737          if (!m.matches()) {
738            throw new IllegalArgumentException(hfileName + " is not a valid HFileLink name!");
739          }
740          linkedTable = TableName.valueOf(m.group(1), m.group(2));
741          linkedRegion = m.group(3);
742          hfileName = m.group(4);
743        }
744        // must create back reference here
745        HFileLink.create(conf, fs, splitDir, familyName, hri.getTable().getNameAsString(),
746          hri.getEncodedName(), linkedTable, linkedRegion, hfileName, true);
747        Path path =
748          new Path(splitDir, HFileLink.createHFileLinkName(linkedTable, linkedRegion, hfileName));
749        LOG.info("Created linkFile:" + path.toString() + " for child: " + hri.getEncodedName()
750          + ", parent: " + regionInfoForFs.getEncodedName());
751        return path;
752      } catch (IOException e) {
753        // if create HFileLink file failed, then just skip the error and create Reference file
754        LOG.error("Create link file for " + hfileName + " for child " + hri.getEncodedName()
755          + "failed, will create Reference file", e);
756      }
757    }
758    // A reference to the bottom half of the hsf store file.
759    Reference r =
760      top ? Reference.createTopReference(splitRow) : Reference.createBottomReference(splitRow);
761    return r.write(fs, p);
762  }
763
764  // ===========================================================================
765  // Merge Helpers
766  // ===========================================================================
767
768  Path getMergesDir(final RegionInfo hri) {
769    return new Path(getTableDir(), hri.getEncodedName());
770  }
771
772  /**
773   * Remove merged region
774   * @param mergedRegion {@link RegionInfo}
775   */
776  public void cleanupMergedRegion(final RegionInfo mergedRegion) throws IOException {
777    Path regionDir = new Path(this.tableDir, mergedRegion.getEncodedName());
778    if (this.fs.exists(regionDir) && !this.fs.delete(regionDir, true)) {
779      throw new IOException("Failed delete of " + regionDir);
780    }
781  }
782
783  static boolean mkdirs(FileSystem fs, Configuration conf, Path dir) throws IOException {
784    if (
785      FSUtils.isDistributedFileSystem(fs)
786        || !conf.getBoolean(HConstants.ENABLE_DATA_FILE_UMASK, false)
787    ) {
788      return fs.mkdirs(dir);
789    }
790    FsPermission perms = CommonFSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
791    return fs.mkdirs(dir, perms);
792  }
793
794  /**
795   * Write out a merge reference under the given merges directory.
796   * @param mergingRegion {@link RegionInfo} for one of the regions being merged.
797   * @param familyName    Column Family Name
798   * @param f             File to create reference.
799   * @return Path to created reference.
800   * @throws IOException if the merge write fails.
801   */
802  public Path mergeStoreFile(RegionInfo mergingRegion, String familyName, HStoreFile f)
803    throws IOException {
804    Path referenceDir = new Path(getMergesDir(regionInfoForFs), familyName);
805    // A whole reference to the store file.
806    Reference r = Reference.createTopReference(mergingRegion.getStartKey());
807    // Add the referred-to regions name as a dot separated suffix.
808    // See REF_NAME_REGEX regex above. The referred-to regions name is
809    // up in the path of the passed in <code>f</code> -- parentdir is family,
810    // then the directory above is the region name.
811    String mergingRegionName = mergingRegion.getEncodedName();
812    // Write reference with same file id only with the other region name as
813    // suffix and into the new region location (under same family).
814    Path p = new Path(referenceDir, f.getPath().getName() + "." + mergingRegionName);
815    return r.write(fs, p);
816  }
817
818  /**
819   * Commit a merged region, making it ready for use.
820   */
821  public void commitMergedRegion(List<Path> allMergedFiles, MasterProcedureEnv env)
822    throws IOException {
823    Path regionDir = getMergesDir(regionInfoForFs);
824    if (regionDir != null && fs.exists(regionDir)) {
825      // Write HRI to a file in case we need to recover hbase:meta
826      Path regionInfoFile = new Path(regionDir, REGION_INFO_FILE);
827      byte[] regionInfoContent = getRegionInfoFileContent(regionInfo);
828      writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
829      insertRegionFilesIntoStoreTracker(allMergedFiles, env, this);
830    }
831  }
832
833  // ===========================================================================
834  // Create/Open/Delete Helpers
835  // ===========================================================================
836
837  /** Returns Content of the file we write out to the filesystem under a region */
838  private static byte[] getRegionInfoFileContent(final RegionInfo hri) throws IOException {
839    return RegionInfo.toDelimitedByteArray(hri);
840  }
841
842  /**
843   * Create a {@link RegionInfo} from the serialized version on-disk.
844   * @param fs        {@link FileSystem} that contains the Region Info file
845   * @param regionDir {@link Path} to the Region Directory that contains the Info file
846   * @return An {@link RegionInfo} instance gotten from the Region Info file.
847   * @throws IOException if an error occurred during file open/read operation.
848   */
849  public static RegionInfo loadRegionInfoFileContent(final FileSystem fs, final Path regionDir)
850    throws IOException {
851    FSDataInputStream in = fs.open(new Path(regionDir, REGION_INFO_FILE));
852    try {
853      return RegionInfo.parseFrom(in);
854    } finally {
855      in.close();
856    }
857  }
858
859  /**
860   * Write the .regioninfo file on-disk.
861   * <p/>
862   * Overwrites if exists already.
863   */
864  private static void writeRegionInfoFileContent(final Configuration conf, final FileSystem fs,
865    final Path regionInfoFile, final byte[] content) throws IOException {
866    // First check to get the permissions
867    FsPermission perms = CommonFSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
868    // Write the RegionInfo file content
869    try (FSDataOutputStream out = FSUtils.create(conf, fs, regionInfoFile, perms, null)) {
870      out.write(content);
871    }
872  }
873
874  /**
875   * Write out an info file under the stored region directory. Useful recovering mangled regions. If
876   * the regionInfo already exists on-disk, then we fast exit.
877   */
878  void checkRegionInfoOnFilesystem() throws IOException {
879    // Compose the content of the file so we can compare to length in filesystem. If not same,
880    // rewrite it (it may have been written in the old format using Writables instead of pb). The
881    // pb version is much shorter -- we write now w/o the toString version -- so checking length
882    // only should be sufficient. I don't want to read the file every time to check if it pb
883    // serialized.
884    byte[] content = getRegionInfoFileContent(regionInfoForFs);
885
886    // Verify if the region directory exists before opening a region. We need to do this since if
887    // the region directory doesn't exist we will re-create the region directory and a new HRI
888    // when HRegion.openHRegion() is called.
889    try {
890      FileStatus status = fs.getFileStatus(getRegionDir());
891    } catch (FileNotFoundException e) {
892      LOG.warn(getRegionDir() + " doesn't exist for region: " + regionInfoForFs.getEncodedName()
893        + " on table " + regionInfo.getTable());
894    }
895
896    try {
897      Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
898      FileStatus status = fs.getFileStatus(regionInfoFile);
899      if (status != null && status.getLen() == content.length) {
900        // Then assume the content good and move on.
901        // NOTE: that the length is not sufficient to define the the content matches.
902        return;
903      }
904
905      LOG.info("Rewriting .regioninfo file at: " + regionInfoFile);
906      if (!fs.delete(regionInfoFile, false)) {
907        throw new IOException("Unable to remove existing " + regionInfoFile);
908      }
909    } catch (FileNotFoundException e) {
910      LOG.warn(REGION_INFO_FILE + " file not found for region: " + regionInfoForFs.getEncodedName()
911        + " on table " + regionInfo.getTable());
912    }
913
914    // Write HRI to a file in case we need to recover hbase:meta
915    writeRegionInfoOnFilesystem(content, true);
916  }
917
918  /**
919   * Write out an info file under the region directory. Useful recovering mangled regions.
920   * @param useTempDir indicate whether or not using the region .tmp dir for a safer file creation.
921   */
922  private void writeRegionInfoOnFilesystem(boolean useTempDir) throws IOException {
923    byte[] content = getRegionInfoFileContent(regionInfoForFs);
924    writeRegionInfoOnFilesystem(content, useTempDir);
925  }
926
927  /**
928   * Write out an info file under the region directory. Useful recovering mangled regions.
929   * @param regionInfoContent serialized version of the {@link RegionInfo}
930   * @param useTempDir        indicate whether or not using the region .tmp dir for a safer file
931   *                          creation.
932   */
933  private void writeRegionInfoOnFilesystem(final byte[] regionInfoContent, final boolean useTempDir)
934    throws IOException {
935    Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
936    if (useTempDir) {
937      // Create in tmpDir and then move into place in case we crash after
938      // create but before close. If we don't successfully close the file,
939      // subsequent region reopens will fail the below because create is
940      // registered in NN.
941
942      // And then create the file
943      Path tmpPath = new Path(getTempDir(), REGION_INFO_FILE);
944
945      // If datanode crashes or if the RS goes down just before the close is called while trying to
946      // close the created regioninfo file in the .tmp directory then on next
947      // creation we will be getting AlreadyCreatedException.
948      // Hence delete and create the file if exists.
949      if (CommonFSUtils.isExists(fs, tmpPath)) {
950        CommonFSUtils.delete(fs, tmpPath, true);
951      }
952
953      // Write HRI to a file in case we need to recover hbase:meta
954      writeRegionInfoFileContent(conf, fs, tmpPath, regionInfoContent);
955
956      // Move the created file to the original path
957      if (fs.exists(tmpPath) && !rename(tmpPath, regionInfoFile)) {
958        throw new IOException("Unable to rename " + tmpPath + " to " + regionInfoFile);
959      }
960    } else {
961      // Write HRI to a file in case we need to recover hbase:meta
962      writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
963    }
964  }
965
966  /**
967   * Create a new Region on file-system.
968   * @param conf       the {@link Configuration} to use
969   * @param fs         {@link FileSystem} from which to add the region
970   * @param tableDir   {@link Path} to where the table is being stored
971   * @param regionInfo {@link RegionInfo} for region to be added
972   * @throws IOException if the region creation fails due to a FileSystem exception.
973   */
974  public static HRegionFileSystem createRegionOnFileSystem(final Configuration conf,
975    final FileSystem fs, final Path tableDir, final RegionInfo regionInfo) throws IOException {
976    HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
977
978    // We only create a .regioninfo and the region directory if this is the default region replica
979    if (regionInfo.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
980      Path regionDir = regionFs.getRegionDir();
981      if (fs.exists(regionDir)) {
982        LOG.warn("Trying to create a region that already exists on disk: " + regionDir);
983      } else {
984        // Create the region directory
985        if (!createDirOnFileSystem(fs, conf, regionDir)) {
986          LOG.warn("Unable to create the region directory: " + regionDir);
987          throw new IOException("Unable to create region directory: " + regionDir);
988        }
989      }
990
991      // Write HRI to a file in case we need to recover hbase:meta
992      regionFs.writeRegionInfoOnFilesystem(false);
993    } else {
994      if (LOG.isDebugEnabled())
995        LOG.debug("Skipping creation of .regioninfo file for " + regionInfo);
996    }
997    return regionFs;
998  }
999
1000  /**
1001   * Open Region from file-system.
1002   * @param conf       the {@link Configuration} to use
1003   * @param fs         {@link FileSystem} from which to add the region
1004   * @param tableDir   {@link Path} to where the table is being stored
1005   * @param regionInfo {@link RegionInfo} for region to be added
1006   * @param readOnly   True if you don't want to edit the region data
1007   * @throws IOException if the region creation fails due to a FileSystem exception.
1008   */
1009  public static HRegionFileSystem openRegionFromFileSystem(final Configuration conf,
1010    final FileSystem fs, final Path tableDir, final RegionInfo regionInfo, boolean readOnly)
1011    throws IOException {
1012    HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
1013    Path regionDir = regionFs.getRegionDir();
1014
1015    if (!fs.exists(regionDir)) {
1016      LOG.warn("Trying to open a region that do not exists on disk: " + regionDir);
1017      throw new IOException("The specified region do not exists on disk: " + regionDir);
1018    }
1019
1020    if (!readOnly) {
1021      // Cleanup temporary directories
1022      regionFs.cleanupTempDir();
1023
1024      // If it doesn't exists, Write HRI to a file, in case we need to recover hbase:meta
1025      // Only create HRI if we are the default replica
1026      if (regionInfo.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
1027        regionFs.checkRegionInfoOnFilesystem();
1028      } else {
1029        if (LOG.isDebugEnabled()) {
1030          LOG.debug("Skipping creation of .regioninfo file for " + regionInfo);
1031        }
1032      }
1033    }
1034
1035    return regionFs;
1036  }
1037
1038  /**
1039   * Remove the region from the table directory, archiving the region's hfiles.
1040   * @param conf       the {@link Configuration} to use
1041   * @param fs         {@link FileSystem} from which to remove the region
1042   * @param tableDir   {@link Path} to where the table is being stored
1043   * @param regionInfo {@link RegionInfo} for region to be deleted
1044   * @throws IOException if the request cannot be completed
1045   */
1046  public static void deleteRegionFromFileSystem(final Configuration conf, final FileSystem fs,
1047    final Path tableDir, final RegionInfo regionInfo) throws IOException {
1048    HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
1049    Path regionDir = regionFs.getRegionDir();
1050
1051    if (!fs.exists(regionDir)) {
1052      LOG.warn("Trying to delete a region that do not exists on disk: " + regionDir);
1053      return;
1054    }
1055
1056    if (LOG.isDebugEnabled()) {
1057      LOG.debug("DELETING region " + regionDir);
1058    }
1059
1060    // Archive region
1061    Path rootDir = CommonFSUtils.getRootDir(conf);
1062    HFileArchiver.archiveRegion(fs, rootDir, tableDir, regionDir);
1063
1064    // Delete empty region dir
1065    if (!fs.delete(regionDir, true)) {
1066      LOG.warn("Failed delete of " + regionDir);
1067    }
1068  }
1069
1070  /**
1071   * Creates a directory. Assumes the user has already checked for this directory existence.
1072   * @return the result of fs.mkdirs(). In case underlying fs throws an IOException, it checks
1073   *         whether the directory exists or not, and returns true if it exists.
1074   */
1075  boolean createDir(Path dir) throws IOException {
1076    int i = 0;
1077    IOException lastIOE = null;
1078    do {
1079      try {
1080        return mkdirs(fs, conf, dir);
1081      } catch (IOException ioe) {
1082        lastIOE = ioe;
1083        if (fs.exists(dir)) return true; // directory is present
1084        try {
1085          sleepBeforeRetry("Create Directory", i + 1);
1086        } catch (InterruptedException e) {
1087          throw (InterruptedIOException) new InterruptedIOException().initCause(e);
1088        }
1089      }
1090    } while (++i <= hdfsClientRetriesNumber);
1091    throw new IOException("Exception in createDir", lastIOE);
1092  }
1093
1094  /**
1095   * Renames a directory. Assumes the user has already checked for this directory existence.
1096   * @return true if rename is successful.
1097   */
1098  boolean rename(Path srcpath, Path dstPath) throws IOException {
1099    IOException lastIOE = null;
1100    int i = 0;
1101    do {
1102      try {
1103        return fs.rename(srcpath, dstPath);
1104      } catch (IOException ioe) {
1105        lastIOE = ioe;
1106        if (!fs.exists(srcpath) && fs.exists(dstPath)) return true; // successful move
1107        // dir is not there, retry after some time.
1108        try {
1109          sleepBeforeRetry("Rename Directory", i + 1);
1110        } catch (InterruptedException e) {
1111          throw (InterruptedIOException) new InterruptedIOException().initCause(e);
1112        }
1113      }
1114    } while (++i <= hdfsClientRetriesNumber);
1115
1116    throw new IOException("Exception in rename", lastIOE);
1117  }
1118
1119  /**
1120   * Deletes a directory. Assumes the user has already checked for this directory existence.
1121   * @return true if the directory is deleted.
1122   */
1123  boolean deleteDir(Path dir) throws IOException {
1124    IOException lastIOE = null;
1125    int i = 0;
1126    do {
1127      try {
1128        return fs.delete(dir, true);
1129      } catch (IOException ioe) {
1130        lastIOE = ioe;
1131        if (!fs.exists(dir)) return true;
1132        // dir is there, retry deleting after some time.
1133        try {
1134          sleepBeforeRetry("Delete Directory", i + 1);
1135        } catch (InterruptedException e) {
1136          throw (InterruptedIOException) new InterruptedIOException().initCause(e);
1137        }
1138      }
1139    } while (++i <= hdfsClientRetriesNumber);
1140
1141    throw new IOException("Exception in DeleteDir", lastIOE);
1142  }
1143
1144  /**
1145   * sleeping logic; handles the interrupt exception.
1146   */
1147  private void sleepBeforeRetry(String msg, int sleepMultiplier) throws InterruptedException {
1148    sleepBeforeRetry(msg, sleepMultiplier, baseSleepBeforeRetries, hdfsClientRetriesNumber);
1149  }
1150
1151  /**
1152   * Creates a directory for a filesystem and configuration object. Assumes the user has already
1153   * checked for this directory existence.
1154   * @return the result of fs.mkdirs(). In case underlying fs throws an IOException, it checks
1155   *         whether the directory exists or not, and returns true if it exists.
1156   */
1157  private static boolean createDirOnFileSystem(FileSystem fs, Configuration conf, Path dir)
1158    throws IOException {
1159    int i = 0;
1160    IOException lastIOE = null;
1161    int hdfsClientRetriesNumber =
1162      conf.getInt("hdfs.client.retries.number", DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
1163    int baseSleepBeforeRetries =
1164      conf.getInt("hdfs.client.sleep.before.retries", DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
1165    do {
1166      try {
1167        return fs.mkdirs(dir);
1168      } catch (IOException ioe) {
1169        lastIOE = ioe;
1170        if (fs.exists(dir)) return true; // directory is present
1171        try {
1172          sleepBeforeRetry("Create Directory", i + 1, baseSleepBeforeRetries,
1173            hdfsClientRetriesNumber);
1174        } catch (InterruptedException e) {
1175          throw (InterruptedIOException) new InterruptedIOException().initCause(e);
1176        }
1177      }
1178    } while (++i <= hdfsClientRetriesNumber);
1179
1180    throw new IOException("Exception in createDir", lastIOE);
1181  }
1182
1183  /**
1184   * sleeping logic for static methods; handles the interrupt exception. Keeping a static version
1185   * for this to avoid re-looking for the integer values.
1186   */
1187  private static void sleepBeforeRetry(String msg, int sleepMultiplier, int baseSleepBeforeRetries,
1188    int hdfsClientRetriesNumber) throws InterruptedException {
1189    if (sleepMultiplier > hdfsClientRetriesNumber) {
1190      if (LOG.isDebugEnabled()) {
1191        LOG.debug(msg + ", retries exhausted");
1192      }
1193      return;
1194    }
1195    if (LOG.isDebugEnabled()) {
1196      LOG.debug(msg + ", sleeping " + baseSleepBeforeRetries + " times " + sleepMultiplier);
1197    }
1198    Thread.sleep((long) baseSleepBeforeRetries * sleepMultiplier);
1199  }
1200}