1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util;
19
20 import java.io.IOException;
21 import java.util.List;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.conf.Configured;
27 import org.apache.hadoop.hbase.Abortable;
28 import org.apache.hadoop.hbase.HBaseConfiguration;
29 import org.apache.hadoop.hbase.HConstants;
30 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
31 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
32 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
33 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.ReplicationPeer;
34 import org.apache.hadoop.hbase.replication.ReplicationStateZKBase;
35 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
36 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
37 import org.apache.hadoop.util.Tool;
38 import org.apache.hadoop.util.ToolRunner;
39 import org.apache.zookeeper.KeeperException;
40 import org.apache.zookeeper.KeeperException.NoNodeException;
41
42
43
44
45 public class ZKDataMigrator extends Configured implements Tool {
46
47 private static final Log LOG = LogFactory.getLog(ZKDataMigrator.class);
48
49 @Override
50 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="REC_CATCH_EXCEPTION",
51 justification="Intended")
52 public int run(String[] as) throws Exception {
53 Configuration conf = getConf();
54 ZooKeeperWatcher zkw = null;
55 try {
56 zkw = new ZooKeeperWatcher(getConf(), "Migrate ZK data to PB.",
57 new ZKDataMigratorAbortable());
58 if (ZKUtil.checkExists(zkw, zkw.baseZNode) == -1) {
59 LOG.info("No hbase related data available in zookeeper. returning..");
60 return 0;
61 }
62 List<String> children = ZKUtil.listChildrenNoWatch(zkw, zkw.baseZNode);
63 if (children == null) {
64 LOG.info("No child nodes to mirgrate. returning..");
65 return 0;
66 }
67 String childPath = null;
68 for (String child : children) {
69 childPath = ZKUtil.joinZNode(zkw.baseZNode, child);
70 if (child.equals(conf.get("zookeeper.znode.rootserver", "root-region-server"))) {
71
72
73 ZKUtil.deleteNodeRecursively(zkw, childPath);
74
75 } else if (child.equals(conf.get("zookeeper.znode.rs", "rs"))) {
76
77
78 ZKUtil.deleteNodeRecursively(zkw, childPath);
79 } else if (child.equals(conf.get("zookeeper.znode.draining.rs", "draining"))) {
80
81
82
83
84 ZKUtil.deleteNodeRecursively(zkw, childPath);
85 } else if (child.equals(conf.get("zookeeper.znode.master", "master"))) {
86
87
88 ZKUtil.deleteNodeRecursively(zkw, childPath);
89 } else if (child.equals(conf.get("zookeeper.znode.backup.masters", "backup-masters"))) {
90
91
92 ZKUtil.deleteNodeRecursively(zkw, childPath);
93 } else if (child.equals(conf.get("zookeeper.znode.state", "shutdown"))) {
94
95
96 ZKUtil.deleteNodeRecursively(zkw, childPath);
97 } else if (child.equals(conf.get("zookeeper.znode.unassigned", "unassigned"))) {
98
99
100
101
102 ZKUtil.deleteNodeRecursively(zkw, childPath);
103 } else if (child.equals(conf.get("zookeeper.znode.tableEnableDisable", "table"))
104 || child.equals(conf.get("zookeeper.znode.masterTableEnableDisable", "table"))) {
105 checkAndMigrateTableStatesToPB(zkw);
106 } else if (child.equals(conf.get("zookeeper.znode.masterTableEnableDisable92",
107 "table92"))) {
108
109
110 ZKUtil.deleteNodeRecursively(zkw, childPath);
111 } else if (child.equals(conf.get("zookeeper.znode.splitlog", "splitlog"))) {
112
113
114 ZKUtil.deleteNodeRecursively(zkw, childPath);
115 } else if (child.equals(conf.get("zookeeper.znode.replication", "replication"))) {
116 checkAndMigrateReplicationNodesToPB(zkw);
117 } else if (child.equals(conf.get("zookeeper.znode.clusterId", "hbaseid"))) {
118
119 ZKUtil.deleteNodeRecursively(zkw, childPath);
120 } else if (child.equals(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION)) {
121
122 ZKUtil.deleteNodeRecursively(zkw, childPath);
123 } else if (child.equals(conf.get("zookeeper.znode.acl.parent", "acl"))) {
124
125 ZKUtil.deleteNodeRecursively(zkw, childPath);
126 }
127 }
128 } catch (Exception e) {
129 LOG.error("Got exception while updating znodes ", e);
130 throw new IOException(e);
131 } finally {
132 if (zkw != null) {
133 zkw.close();
134 }
135 }
136 return 0;
137 }
138
139 private void checkAndMigrateTableStatesToPB(ZooKeeperWatcher zkw) throws KeeperException,
140 InterruptedException {
141 List<String> tables = ZKUtil.listChildrenNoWatch(zkw, zkw.tableZNode);
142 if (tables == null) {
143 LOG.info("No table present to migrate table state to PB. returning..");
144 return;
145 }
146 for (String table : tables) {
147 String znode = ZKUtil.joinZNode(zkw.tableZNode, table);
148
149
150 if (table.equals("-ROOT-") || table.equals(".META.")) {
151 ZKUtil.deleteNode(zkw, znode);
152 continue;
153 }
154 byte[] data = ZKUtil.getData(zkw, znode);
155 if (ProtobufUtil.isPBMagicPrefix(data)) continue;
156 ZooKeeperProtos.Table.Builder builder = ZooKeeperProtos.Table.newBuilder();
157 builder.setState(ZooKeeperProtos.Table.State.valueOf(Bytes.toString(data)));
158 data = ProtobufUtil.prependPBMagic(builder.build().toByteArray());
159 ZKUtil.setData(zkw, znode, data);
160 }
161 }
162
163 private void checkAndMigrateReplicationNodesToPB(ZooKeeperWatcher zkw) throws KeeperException,
164 InterruptedException {
165 String replicationZnodeName = getConf().get("zookeeper.znode.replication", "replication");
166 String replicationPath = ZKUtil.joinZNode(zkw.baseZNode, replicationZnodeName);
167 List<String> replicationZnodes = ZKUtil.listChildrenNoWatch(zkw, replicationPath);
168 if (replicationZnodes == null) {
169 LOG.info("No replication related znodes present to migrate. returning..");
170 return;
171 }
172 for (String child : replicationZnodes) {
173 String znode = ZKUtil.joinZNode(replicationPath, child);
174 if (child.equals(getConf().get("zookeeper.znode.replication.peers", "peers"))) {
175 List<String> peers = ZKUtil.listChildrenNoWatch(zkw, znode);
176 if (peers == null || peers.isEmpty()) {
177 LOG.info("No peers present to migrate. returning..");
178 continue;
179 }
180 checkAndMigratePeerZnodesToPB(zkw, znode, peers);
181 } else if (child.equals(getConf().get("zookeeper.znode.replication.state", "state"))) {
182
183 ZKUtil.deleteNodeRecursively(zkw, znode);
184 } else if (child.equals(getConf().get("zookeeper.znode.replication.rs", "rs"))) {
185 List<String> rsList = ZKUtil.listChildrenNoWatch(zkw, znode);
186 if (rsList == null || rsList.isEmpty()) continue;
187 for (String rs : rsList) {
188 checkAndMigrateQueuesToPB(zkw, znode, rs);
189 }
190 }
191 }
192 }
193
194 private void checkAndMigrateQueuesToPB(ZooKeeperWatcher zkw, String znode, String rs)
195 throws KeeperException, NoNodeException, InterruptedException {
196 String rsPath = ZKUtil.joinZNode(znode, rs);
197 List<String> peers = ZKUtil.listChildrenNoWatch(zkw, rsPath);
198 if (peers == null || peers.isEmpty()) return;
199 String peerPath = null;
200 for (String peer : peers) {
201 peerPath = ZKUtil.joinZNode(rsPath, peer);
202 List<String> files = ZKUtil.listChildrenNoWatch(zkw, peerPath);
203 if (files == null || files.isEmpty()) continue;
204 String filePath = null;
205 for (String file : files) {
206 filePath = ZKUtil.joinZNode(peerPath, file);
207 byte[] data = ZKUtil.getData(zkw, filePath);
208 if (data == null || Bytes.equals(data, HConstants.EMPTY_BYTE_ARRAY)) continue;
209 if (ProtobufUtil.isPBMagicPrefix(data)) continue;
210 ZKUtil.setData(zkw, filePath,
211 ZKUtil.positionToByteArray(Long.parseLong(Bytes.toString(data))));
212 }
213 }
214 }
215
216 private void checkAndMigratePeerZnodesToPB(ZooKeeperWatcher zkw, String znode,
217 List<String> peers) throws KeeperException, NoNodeException, InterruptedException {
218 for (String peer : peers) {
219 String peerZnode = ZKUtil.joinZNode(znode, peer);
220 byte[] data = ZKUtil.getData(zkw, peerZnode);
221 if (!ProtobufUtil.isPBMagicPrefix(data)) {
222 migrateClusterKeyToPB(zkw, peerZnode, data);
223 }
224 String peerStatePath = ZKUtil.joinZNode(peerZnode,
225 getConf().get("zookeeper.znode.replication.peers.state", "peer-state"));
226 if (ZKUtil.checkExists(zkw, peerStatePath) != -1) {
227 data = ZKUtil.getData(zkw, peerStatePath);
228 if (ProtobufUtil.isPBMagicPrefix(data)) continue;
229 migratePeerStateToPB(zkw, data, peerStatePath);
230 }
231 }
232 }
233
234 private void migrateClusterKeyToPB(ZooKeeperWatcher zkw, String peerZnode, byte[] data)
235 throws KeeperException, NoNodeException {
236 ReplicationPeer peer = ZooKeeperProtos.ReplicationPeer.newBuilder()
237 .setClusterkey(Bytes.toString(data)).build();
238 ZKUtil.setData(zkw, peerZnode, ProtobufUtil.prependPBMagic(peer.toByteArray()));
239 }
240
241 private void migratePeerStateToPB(ZooKeeperWatcher zkw, byte[] data,
242 String peerStatePath)
243 throws KeeperException, NoNodeException {
244 String state = Bytes.toString(data);
245 if (ZooKeeperProtos.ReplicationState.State.ENABLED.name().equals(state)) {
246 ZKUtil.setData(zkw, peerStatePath, ReplicationStateZKBase.ENABLED_ZNODE_BYTES);
247 } else if (ZooKeeperProtos.ReplicationState.State.DISABLED.name().equals(state)) {
248 ZKUtil.setData(zkw, peerStatePath, ReplicationStateZKBase.DISABLED_ZNODE_BYTES);
249 }
250 }
251
252 public static void main(String args[]) throws Exception {
253 System.exit(ToolRunner.run(HBaseConfiguration.create(), new ZKDataMigrator(), args));
254 }
255
256 static class ZKDataMigratorAbortable implements Abortable {
257 private boolean aborted = false;
258
259 @Override
260 public void abort(String why, Throwable e) {
261 LOG.error("Got aborted with reason: " + why + ", and error: " + e);
262 this.aborted = true;
263 }
264
265 @Override
266 public boolean isAborted() {
267 return this.aborted;
268 }
269 }
270 }