View Javadoc

1   /**
2     * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.zookeeper;
19  
20  import java.io.IOException;
21  import java.io.InterruptedIOException;
22  import java.io.UnsupportedEncodingException;
23  import java.net.URLDecoder;
24  import java.net.URLEncoder;
25  import java.util.List;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.fs.FileSystem;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.classification.InterfaceAudience;
33  import org.apache.hadoop.hbase.exceptions.DeserializationException;
34  import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionStoreSequenceIds;
35  import org.apache.zookeeper.KeeperException;
36  
37  /**
38   * Common methods and attributes used by {@link org.apache.hadoop.hbase.master.SplitLogManager} 
39   * and {@link org.apache.hadoop.hbase.regionserver.SplitLogWorker}
40   * running distributed splitting of WAL logs.
41   */
42  @InterfaceAudience.Private
43  public class ZKSplitLog {
44    private static final Log LOG = LogFactory.getLog(ZKSplitLog.class);
45  
46    /**
47     * Gets the full path node name for the log file being split.
48     * This method will url encode the filename.
49     * @param zkw zk reference
50     * @param filename log file name (only the basename)
51     */
52    public static String getEncodedNodeName(ZooKeeperWatcher zkw, String filename) {
53      return ZKUtil.joinZNode(zkw.splitLogZNode, encode(filename));
54    }
55  
56    public static String getFileName(String node) {
57      String basename = node.substring(node.lastIndexOf('/') + 1);
58      return decode(basename);
59    }
60  
61    static String encode(String s) {
62      try {
63        return URLEncoder.encode(s, "UTF-8");
64      } catch (UnsupportedEncodingException e) {
65        throw new RuntimeException("URLENCODER doesn't support UTF-8");
66      }
67    }
68  
69    static String decode(String s) {
70      try {
71        return URLDecoder.decode(s, "UTF-8");
72      } catch (UnsupportedEncodingException e) {
73        throw new RuntimeException("URLDecoder doesn't support UTF-8");
74      }
75    }
76  
77    public static String getRescanNode(ZooKeeperWatcher zkw) {
78      return ZKUtil.joinZNode(zkw.splitLogZNode, "RESCAN");
79    }
80  
81    /**
82     * @param name the last part in path
83     * @return whether the node name represents a rescan node
84     */
85    public static boolean isRescanNode(String name) {
86      return name.startsWith("RESCAN");
87    }
88  
89    /**
90     * @param zkw
91     * @param path the absolute path, starts with '/'
92     * @return whether the path represents a rescan node
93     */
94    public static boolean isRescanNode(ZooKeeperWatcher zkw, String path) {
95      String prefix = getRescanNode(zkw);
96      if (path.length() <= prefix.length()) {
97        return false;
98      }
99      for (int i = 0; i < prefix.length(); i++) {
100       if (prefix.charAt(i) != path.charAt(i)) {
101         return false;
102       }
103     }
104     return true;
105   }
106 
107   public static boolean isTaskPath(ZooKeeperWatcher zkw, String path) {
108     String dirname = path.substring(0, path.lastIndexOf('/'));
109     return dirname.equals(zkw.splitLogZNode);
110   }
111 
112   public static Path getSplitLogDir(Path rootdir, String tmpname) {
113     return new Path(new Path(rootdir, HConstants.SPLIT_LOGDIR_NAME), tmpname);
114   }
115 
116 
117   public static String getSplitLogDirTmpComponent(final String worker, String file) {
118     return worker + "_" + ZKSplitLog.encode(file);
119   }
120 
121   public static void markCorrupted(Path rootdir, String logFileName,
122       FileSystem fs) {
123     Path file = new Path(getSplitLogDir(rootdir, logFileName), "corrupt");
124     try {
125       fs.createNewFile(file);
126     } catch (IOException e) {
127       LOG.warn("Could not flag a log file as corrupted. Failed to create " +
128           file, e);
129     }
130   }
131 
132   public static boolean isCorrupted(Path rootdir, String logFileName,
133       FileSystem fs) throws IOException {
134     Path file = new Path(getSplitLogDir(rootdir, logFileName), "corrupt");
135     boolean isCorrupt;
136     isCorrupt = fs.exists(file);
137     return isCorrupt;
138   }
139 
140   /*
141    * Following methods come from SplitLogManager
142    */
143 
144   /**
145    * check if /hbase/recovering-regions/<current region encoded name> exists. Returns true if exists
146    * and set watcher as well.
147    * @param zkw
148    * @param regionEncodedName region encode name
149    * @return true when /hbase/recovering-regions/<current region encoded name> exists
150    * @throws KeeperException
151    */
152   public static boolean
153       isRegionMarkedRecoveringInZK(ZooKeeperWatcher zkw, String regionEncodedName)
154           throws KeeperException {
155     boolean result = false;
156     String nodePath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, regionEncodedName);
157 
158     byte[] node = ZKUtil.getDataAndWatch(zkw, nodePath);
159     if (node != null) {
160       result = true;
161     }
162     return result;
163   }
164 
165   /**
166    * @param bytes - Content of a failed region server or recovering region znode.
167    * @return long - The last flushed sequence Id for the region server
168    */
169   public static long parseLastFlushedSequenceIdFrom(final byte[] bytes) {
170     long lastRecordedFlushedSequenceId = -1l;
171     try {
172       lastRecordedFlushedSequenceId = ZKUtil.parseWALPositionFrom(bytes);
173     } catch (DeserializationException e) {
174       lastRecordedFlushedSequenceId = -1l;
175       LOG.warn("Can't parse last flushed sequence Id", e);
176     }
177     return lastRecordedFlushedSequenceId;
178   }
179 
180   public static void deleteRecoveringRegionZNodes(ZooKeeperWatcher watcher, List<String> regions) {
181     try {
182       if (regions == null) {
183         // remove all children under /home/recovering-regions
184         LOG.debug("Garbage collecting all recovering region znodes");
185         ZKUtil.deleteChildrenRecursively(watcher, watcher.recoveringRegionsZNode);
186       } else {
187         for (String curRegion : regions) {
188           String nodePath = ZKUtil.joinZNode(watcher.recoveringRegionsZNode, curRegion);
189           ZKUtil.deleteNodeRecursively(watcher, nodePath);
190         }
191       }
192     } catch (KeeperException e) {
193       LOG.warn("Cannot remove recovering regions from ZooKeeper", e);
194     }
195   }
196 
197   /**
198    * This function is used in distributedLogReplay to fetch last flushed sequence id from ZK
199    * @param zkw
200    * @param serverName
201    * @param encodedRegionName
202    * @return the last flushed sequence ids recorded in ZK of the region for <code>serverName<code>
203    * @throws IOException
204    */
205 
206   public static RegionStoreSequenceIds getRegionFlushedSequenceId(ZooKeeperWatcher zkw,
207       String serverName, String encodedRegionName) throws IOException {
208     // when SplitLogWorker recovers a region by directly replaying unflushed WAL edits,
209     // last flushed sequence Id changes when newly assigned RS flushes writes to the region.
210     // If the newly assigned RS fails again(a chained RS failures scenario), the last flushed
211     // sequence Id name space (sequence Id only valid for a particular RS instance), changes
212     // when different newly assigned RS flushes the region.
213     // Therefore, in this mode we need to fetch last sequence Ids from ZK where we keep history of
214     // last flushed sequence Id for each failed RS instance.
215     RegionStoreSequenceIds result = null;
216     String nodePath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, encodedRegionName);
217     nodePath = ZKUtil.joinZNode(nodePath, serverName);
218     try {
219       byte[] data;
220       try {
221         data = ZKUtil.getData(zkw, nodePath);
222       } catch (InterruptedException e) {
223         throw new InterruptedIOException();
224       }
225       if (data != null) {
226         result = ZKUtil.parseRegionStoreSequenceIds(data);
227       }
228     } catch (KeeperException e) {
229       throw new IOException("Cannot get lastFlushedSequenceId from ZooKeeper for server="
230           + serverName + "; region=" + encodedRegionName, e);
231     } catch (DeserializationException e) {
232       LOG.warn("Can't parse last flushed sequence Id from znode:" + nodePath, e);
233     }
234     return result;
235   }
236 }