View Javadoc

1   /**
2     * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.zookeeper;
19  
20  import java.io.IOException;
21  import java.io.InterruptedIOException;
22  import java.io.UnsupportedEncodingException;
23  import java.net.URLDecoder;
24  import java.net.URLEncoder;
25  import java.util.List;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.fs.FileSystem;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.classification.InterfaceAudience;
33  import org.apache.hadoop.hbase.exceptions.DeserializationException;
34  import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionStoreSequenceIds;
35  import org.apache.zookeeper.KeeperException;
36  
37  /**
38   * Common methods and attributes used by 
39   * {@link org.apache.hadoop.hbase.master.SplitLogManager} and 
40   * {@link org.apache.hadoop.hbase.regionserver.SplitLogWorker}
41   * running distributed splitting of WAL logs.
42   */
43  @InterfaceAudience.Private
44  public class ZKSplitLog {
45    private static final Log LOG = LogFactory.getLog(ZKSplitLog.class);
46  
47    /**
48     * Gets the full path node name for the log file being split.
49     * This method will url encode the filename.
50     * @param zkw zk reference
51     * @param filename log file name (only the basename)
52     */
53    public static String getEncodedNodeName(ZooKeeperWatcher zkw, String filename) {
54      return ZKUtil.joinZNode(zkw.splitLogZNode, encode(filename));
55    }
56  
57    public static String getFileName(String node) {
58      String basename = node.substring(node.lastIndexOf('/') + 1);
59      return decode(basename);
60    }
61  
62    static String encode(String s) {
63      try {
64        return URLEncoder.encode(s, "UTF-8");
65      } catch (UnsupportedEncodingException e) {
66        throw new RuntimeException("URLENCODER doesn't support UTF-8");
67      }
68    }
69  
70    static String decode(String s) {
71      try {
72        return URLDecoder.decode(s, "UTF-8");
73      } catch (UnsupportedEncodingException e) {
74        throw new RuntimeException("URLDecoder doesn't support UTF-8");
75      }
76    }
77  
78    public static String getRescanNode(ZooKeeperWatcher zkw) {
79      return ZKUtil.joinZNode(zkw.splitLogZNode, "RESCAN");
80    }
81  
82    /**
83     * @param name the last part in path
84     * @return whether the node name represents a rescan node
85     */
86    public static boolean isRescanNode(String name) {
87      return name.startsWith("RESCAN");
88    }
89  
90    /**
91     * @param zkw
92     * @param path the absolute path, starts with '/'
93     * @return whether the path represents a rescan node
94     */
95    public static boolean isRescanNode(ZooKeeperWatcher zkw, String path) {
96      String prefix = getRescanNode(zkw);
97      if (path.length() <= prefix.length()) {
98        return false;
99      }
100     for (int i = 0; i < prefix.length(); i++) {
101       if (prefix.charAt(i) != path.charAt(i)) {
102         return false;
103       }
104     }
105     return true;
106   }
107 
108   public static boolean isTaskPath(ZooKeeperWatcher zkw, String path) {
109     String dirname = path.substring(0, path.lastIndexOf('/'));
110     return dirname.equals(zkw.splitLogZNode);
111   }
112 
113   public static Path getSplitLogDir(Path rootdir, String tmpname) {
114     return new Path(new Path(rootdir, HConstants.SPLIT_LOGDIR_NAME), tmpname);
115   }
116 
117 
118   public static String getSplitLogDirTmpComponent(final String worker, String file) {
119     return worker + "_" + ZKSplitLog.encode(file);
120   }
121 
122   public static void markCorrupted(Path rootdir, String logFileName,
123       FileSystem fs) {
124     Path file = new Path(getSplitLogDir(rootdir, logFileName), "corrupt");
125     try {
126       fs.createNewFile(file);
127     } catch (IOException e) {
128       LOG.warn("Could not flag a log file as corrupted. Failed to create " +
129           file, e);
130     }
131   }
132 
133   public static boolean isCorrupted(Path rootdir, String logFileName,
134       FileSystem fs) throws IOException {
135     Path file = new Path(getSplitLogDir(rootdir, logFileName), "corrupt");
136     boolean isCorrupt;
137     isCorrupt = fs.exists(file);
138     return isCorrupt;
139   }
140 
141   /*
142    * Following methods come from SplitLogManager
143    */
144 
145   /**
146    * check if /hbase/recovering-regions/&lt;current region encoded name&gt;
147    * exists. Returns true if exists and set watcher as well.
148    * @param zkw
149    * @param regionEncodedName region encode name
150    * @return true when /hbase/recovering-regions/&lt;current region encoded name&gt; exists
151    * @throws KeeperException
152    */
153   public static boolean
154       isRegionMarkedRecoveringInZK(ZooKeeperWatcher zkw, String regionEncodedName)
155           throws KeeperException {
156     boolean result = false;
157     String nodePath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, regionEncodedName);
158 
159     byte[] node = ZKUtil.getDataAndWatch(zkw, nodePath);
160     if (node != null) {
161       result = true;
162     }
163     return result;
164   }
165 
166   /**
167    * @param bytes - Content of a failed region server or recovering region znode.
168    * @return long - The last flushed sequence Id for the region server
169    */
170   public static long parseLastFlushedSequenceIdFrom(final byte[] bytes) {
171     long lastRecordedFlushedSequenceId = -1l;
172     try {
173       lastRecordedFlushedSequenceId = ZKUtil.parseWALPositionFrom(bytes);
174     } catch (DeserializationException e) {
175       lastRecordedFlushedSequenceId = -1l;
176       LOG.warn("Can't parse last flushed sequence Id", e);
177     }
178     return lastRecordedFlushedSequenceId;
179   }
180 
181   public static void deleteRecoveringRegionZNodes(ZooKeeperWatcher watcher, List<String> regions) {
182     try {
183       if (regions == null) {
184         // remove all children under /home/recovering-regions
185         LOG.debug("Garbage collecting all recovering region znodes");
186         ZKUtil.deleteChildrenRecursively(watcher, watcher.recoveringRegionsZNode);
187       } else {
188         for (String curRegion : regions) {
189           String nodePath = ZKUtil.joinZNode(watcher.recoveringRegionsZNode, curRegion);
190           ZKUtil.deleteNodeRecursively(watcher, nodePath);
191         }
192       }
193     } catch (KeeperException e) {
194       LOG.warn("Cannot remove recovering regions from ZooKeeper", e);
195     }
196   }
197 
198   /**
199    * This function is used in distributedLogReplay to fetch last flushed sequence id from ZK
200    * @param zkw
201    * @param serverName
202    * @param encodedRegionName
203    * @return the last flushed sequence ids recorded in ZK of the region for <code>serverName</code>
204    * @throws IOException
205    */
206 
207   public static RegionStoreSequenceIds getRegionFlushedSequenceId(ZooKeeperWatcher zkw,
208       String serverName, String encodedRegionName) throws IOException {
209     // when SplitLogWorker recovers a region by directly replaying unflushed WAL edits,
210     // last flushed sequence Id changes when newly assigned RS flushes writes to the region.
211     // If the newly assigned RS fails again(a chained RS failures scenario), the last flushed
212     // sequence Id name space (sequence Id only valid for a particular RS instance), changes
213     // when different newly assigned RS flushes the region.
214     // Therefore, in this mode we need to fetch last sequence Ids from ZK where we keep history of
215     // last flushed sequence Id for each failed RS instance.
216     RegionStoreSequenceIds result = null;
217     String nodePath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, encodedRegionName);
218     nodePath = ZKUtil.joinZNode(nodePath, serverName);
219     try {
220       byte[] data;
221       try {
222         data = ZKUtil.getData(zkw, nodePath);
223       } catch (InterruptedException e) {
224         throw new InterruptedIOException();
225       }
226       if (data != null) {
227         result = ZKUtil.parseRegionStoreSequenceIds(data);
228       }
229     } catch (KeeperException e) {
230       throw new IOException("Cannot get lastFlushedSequenceId from ZooKeeper for server="
231           + serverName + "; region=" + encodedRegionName, e);
232     } catch (DeserializationException e) {
233       LOG.warn("Can't parse last flushed sequence Id from znode:" + nodePath, e);
234     }
235     return result;
236   }
237 }