View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase;
19  
20  import java.io.IOException;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.fs.FSDataOutputStream;
26  import org.apache.hadoop.fs.FileSystem;
27  import org.apache.hadoop.fs.Path;
28  import org.apache.hadoop.fs.permission.FsPermission;
29  import org.apache.hadoop.hbase.regionserver.wal.HLogFileSystem;
30  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
31  import org.apache.hadoop.hbase.util.FSUtils;
32  import org.apache.hadoop.hbase.util.Threads;
33  
34  /**
35   * An abstraction of the underlying filesystem. This is used by other entities such as
36   * {@link HLogFileSystem}, to make calls to the underlying filesystem.
37   *
38   */
39  public abstract class HBaseFileSystem {
40  
41    public static final Log LOG = LogFactory.getLog(HBaseFileSystem.class);
42  
43    /**
44     * In order to handle NN connectivity hiccups, one need to retry non-idempotent operation at the
45     * client level.
46     */
47    protected static int hdfsClientRetriesNumber;
48    private static int baseSleepBeforeRetries;
49    private static final int DEFAULT_HDFS_CLIENT_RETRIES_NUMBER = 10;
50    private static final int DEFAULT_BASE_SLEEP_BEFORE_RETRIES = 1000;
51    // This static block is added for performance reasons. This is to ensure we are not checking
52    // in the method calls whether retry properties are set or not. Refer to HBase-8288 for more 
53    // context.
54    static {
55      setRetryCounts(HBaseConfiguration.create());
56    }
57  
58    /**
59     * Deletes a file. Assumes the user has already checked for this file's existence.
60     * @param fs
61     * @param file
62     * @return true if the file is deleted.
63     * @throws IOException
64     */
65    public static boolean deleteFileFromFileSystem(FileSystem fs, Path file)
66        throws IOException {
67      IOException lastIOE = null;
68      int i = 0;
69      do {
70        try {
71          return fs.delete(file, false);
72        } catch (IOException ioe) {
73          lastIOE = ioe;
74          if (!fs.exists(file)) return true;
75          // dir is there, retry deleting after some time.
76          sleepBeforeRetry("Delete File", i + 1);
77        }
78      } while (++i <= hdfsClientRetriesNumber);
79      throw new IOException("Exception in deleteFileFromFileSystem", lastIOE);
80    }
81    
82    
83    /**
84     * Deletes a directory. Assumes the user has already checked for this directory's existence.
85     * @param fs
86     * @param dir
87     * @return true if the directory is deleted.
88     * @throws IOException
89     */
90    public static boolean deleteDirFromFileSystem(FileSystem fs, Path dir)
91        throws IOException {
92      IOException lastIOE = null;
93      int i = 0;
94      do {
95        try {
96          return fs.delete(dir, true);
97        } catch (IOException ioe) {
98          lastIOE = ioe;
99          if (!fs.exists(dir)) return true;
100         // dir is there, retry deleting after some time.
101         sleepBeforeRetry("Delete Dir", i + 1);
102       }
103     } while (++i <= hdfsClientRetriesNumber);
104     throw new IOException("Exception in deleteDirFromFileSystem", lastIOE);
105   }
106 
107   protected static void setRetryCounts(Configuration conf) {
108       hdfsClientRetriesNumber = conf.getInt("hdfs.client.retries.number",
109         DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
110       baseSleepBeforeRetries = conf.getInt("hdfs.client.sleep.before.retries",
111         DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
112   }
113   
114   /**
115    * Creates a directory for a filesystem and configuration object. Assumes the user has already
116    * checked for this directory existence.
117    * @param fs
118    * @param dir
119    * @return the result of fs.mkdirs(). In case underlying fs throws an IOException, it checks
120    *         whether the directory exists or not, and returns true if it exists.
121    * @throws IOException
122    */
123   public static boolean makeDirOnFileSystem(FileSystem fs, Path dir)
124       throws IOException {
125     int i = 0;
126     IOException lastIOE = null;
127     do {
128       try {
129         return fs.mkdirs(dir);
130       } catch (IOException ioe) {
131         lastIOE = ioe;
132         if (fs.exists(dir)) return true; // directory is present
133         sleepBeforeRetry("Create Directory", i+1);
134       }
135     } while (++i <= hdfsClientRetriesNumber);
136     throw new IOException("Exception in makeDirOnFileSystem", lastIOE);
137   }
138   
139   /**
140    * Renames a directory. Assumes the user has already checked for this directory existence.
141    * @param fs
142    * @param src
143    * @param dst
144    * @return true if the directory is renamed.
145    * @throws IOException
146    */
147   public static boolean renameDirForFileSystem(FileSystem fs, Path src, Path dst)
148       throws IOException {
149     IOException lastIOE = null;
150     int i = 0;
151     do {
152       try {
153         return fs.rename(src, dst);
154       } catch (IOException ioe) {
155         lastIOE = ioe;
156         if (!fs.exists(src) && fs.exists(dst)) return true;
157         // src is there, retry renaming after some time.
158         sleepBeforeRetry("Rename Directory", i + 1);
159       }
160     } while (++i <= hdfsClientRetriesNumber);
161     throw new IOException("Exception in renameDirForFileSystem", lastIOE);
162   }
163 
164   /**
165    * Creates a path on the file system. Checks whether the path exists already or not, and use it
166    * for retrying in case underlying fs throws an exception. 
167    * If the dir already exists and overwrite flag is false, the underlying FileSystem throws
168    *  an IOE. It is not retried and the IOE is re-thrown to the caller.
169    * @param fs
170    * @param dir
171    * @param overwrite
172    * @return
173    * @throws IOException
174    */
175   public static FSDataOutputStream createPathOnFileSystem(FileSystem fs, Path dir,
176       boolean overwrite) throws IOException {
177     int i = 0;
178     boolean existsBefore = fs.exists(dir);
179     IOException lastIOE = null;
180     do {
181       try {
182         return fs.create(dir, overwrite);
183       } catch (IOException ioe) {
184         lastIOE = ioe;
185         if (existsBefore && !overwrite) throw ioe;// a legitimate exception
186         sleepBeforeRetry("Create Path", i + 1);
187       }
188     } while (++i <= hdfsClientRetriesNumber);
189     throw new IOException("Exception in createPathOnFileSystem", lastIOE);
190   }
191 
192   /**
193    * Creates the specified file with the given permission. 
194    * If the dir already exists and the overwrite flag is false, underlying FileSystem throws
195    * an IOE. It is not retried and the IOE is re-thrown to the caller.
196    * @param fs
197    * @param path
198    * @param perm
199    * @param overwrite
200    * @return
201    * @throws IOException
202    */
203   public static FSDataOutputStream createPathWithPermsOnFileSystem(FileSystem fs,
204       Path path, FsPermission perm, boolean overwrite) throws IOException {
205     int i = 0;
206     IOException lastIOE = null;
207     boolean existsBefore = fs.exists(path);
208     do {
209       try {
210         return fs.create(path, perm, overwrite, FSUtils.getDefaultBufferSize(fs),
211           FSUtils.getDefaultReplication(fs, path), FSUtils.getDefaultBlockSize(fs, path), null);
212       } catch (IOException ioe) {
213         lastIOE = ioe;
214         if (existsBefore && !overwrite) throw ioe;// a legitimate exception
215         sleepBeforeRetry("Create Path with Perms", i + 1);
216       }
217     } while (++i <= hdfsClientRetriesNumber);
218     throw new IOException("Exception in createPathWithPermsOnFileSystem", lastIOE);
219   }
220 
221 /**
222  * Creates the file. Assumes the user has already checked for this file existence.
223  * @param fs
224  * @param dir
225  * @return result true if the file is created with this call, false otherwise.
226  * @throws IOException
227  */
228   public static boolean createNewFileOnFileSystem(FileSystem fs, Path file)
229       throws IOException {
230     int i = 0;
231     IOException lastIOE = null;
232     do {
233       try {
234         return fs.createNewFile(file);
235       } catch (IOException ioe) {
236         lastIOE = ioe;
237         if (fs.exists(file)) return true; // file exists now, return true.
238         sleepBeforeRetry("Create NewFile", i + 1);
239       }
240     } while (++i <= hdfsClientRetriesNumber);
241     throw new IOException("Exception in createNewFileOnFileSystem", lastIOE);
242   }
243   
244   /**
245    * sleeping logic for static methods; handles the interrupt exception. Keeping a static version
246    * for this to avoid re-looking for the integer values.
247    */
248   protected static void sleepBeforeRetry(String msg, int sleepMultiplier) {
249     if (sleepMultiplier > hdfsClientRetriesNumber) {
250       LOG.warn(msg + ", retries exhausted");
251       return;
252     }
253     LOG.info(msg + ", sleeping " + baseSleepBeforeRetries + " times " + sleepMultiplier);
254     Threads.sleep(baseSleepBeforeRetries * sleepMultiplier);
255   }
256   
257   /**
258    * rename the src path to dest path and set the dest path's modify time to current timestamp
259    */
260   public static boolean renameAndSetModifyTime(final FileSystem fs, Path src, Path dest)
261       throws IOException {
262     // set the modify time for TimeToLive Cleaner
263     fs.setTimes(src, EnvironmentEdgeManager.currentTimeMillis(), -1);
264     return renameDirForFileSystem(fs, src, dest);
265   }
266 }