1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.procedure;
19
20 import java.io.IOException;
21 import java.util.Arrays;
22 import java.util.List;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.hbase.classification.InterfaceAudience;
27 import org.apache.hadoop.hbase.errorhandling.ForeignException;
28 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
29 import org.apache.hadoop.hbase.util.Bytes;
30 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
31 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
32 import org.apache.zookeeper.KeeperException;
33
34 import com.google.protobuf.InvalidProtocolBufferException;
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54 @InterfaceAudience.Private
55 public class ZKProcedureMemberRpcs implements ProcedureMemberRpcs {
56 private static final Log LOG = LogFactory.getLog(ZKProcedureMemberRpcs.class);
57
58 private final ZKProcedureUtil zkController;
59
60 protected ProcedureMember member;
61 private String memberName;
62
63
64
65
66
67
68
69
70 public ZKProcedureMemberRpcs(final ZooKeeperWatcher watcher, final String procType)
71 throws KeeperException {
72 this.zkController = new ZKProcedureUtil(watcher, procType) {
73 @Override
74 public void nodeCreated(String path) {
75 if (!isInProcedurePath(path)) {
76 return;
77 }
78
79 LOG.info("Received created event:" + path);
80
81 if (isAcquiredNode(path)) {
82 waitForNewProcedures();
83 return;
84 } else if (isAbortNode(path)) {
85 watchForAbortedProcedures();
86 return;
87 }
88 String parent = ZKUtil.getParent(path);
89
90 if (isReachedNode(parent)) {
91 receivedReachedGlobalBarrier(path);
92 return;
93 } else if (isAbortNode(parent)) {
94 abort(path);
95 return;
96 } else if (isAcquiredNode(parent)) {
97 startNewSubprocedure(path);
98 } else {
99 LOG.debug("Ignoring created notification for node:" + path);
100 }
101 }
102
103 @Override
104 public void nodeChildrenChanged(String path) {
105 if (path.equals(this.acquiredZnode)) {
106 LOG.info("Received procedure start children changed event: " + path);
107 waitForNewProcedures();
108 } else if (path.equals(this.abortZnode)) {
109 LOG.info("Received procedure abort children changed event: " + path);
110 watchForAbortedProcedures();
111 }
112 }
113 };
114 }
115
116 public ZKProcedureUtil getZkController() {
117 return zkController;
118 }
119
120 @Override
121 public String getMemberName() {
122 return memberName;
123 }
124
125
126
127
128
129 private void receivedReachedGlobalBarrier(String path) {
130 LOG.debug("Recieved reached global barrier:" + path);
131 String procName = ZKUtil.getNodeName(path);
132 this.member.receivedReachedGlobalBarrier(procName);
133 }
134
135 private void watchForAbortedProcedures() {
136 LOG.debug("Checking for aborted procedures on node: '" + zkController.getAbortZnode() + "'");
137 try {
138
139 for (String node : ZKUtil.listChildrenAndWatchForNewChildren(zkController.getWatcher(),
140 zkController.getAbortZnode())) {
141 String abortNode = ZKUtil.joinZNode(zkController.getAbortZnode(), node);
142 abort(abortNode);
143 }
144 } catch (KeeperException e) {
145 member.controllerConnectionFailure("Failed to list children for abort node:"
146 + zkController.getAbortZnode(), new IOException(e));
147 }
148 }
149
150 private void waitForNewProcedures() {
151
152 LOG.debug("Looking for new procedures under znode:'" + zkController.getAcquiredBarrier() + "'");
153 List<String> runningProcedures = null;
154 try {
155 runningProcedures = ZKUtil.listChildrenAndWatchForNewChildren(zkController.getWatcher(),
156 zkController.getAcquiredBarrier());
157 if (runningProcedures == null) {
158 LOG.debug("No running procedures.");
159 return;
160 }
161 } catch (KeeperException e) {
162 member.controllerConnectionFailure("General failure when watching for new procedures",
163 new IOException(e));
164 }
165 if (runningProcedures == null) {
166 LOG.debug("No running procedures.");
167 return;
168 }
169 for (String procName : runningProcedures) {
170
171 String path = ZKUtil.joinZNode(zkController.getAcquiredBarrier(), procName);
172 startNewSubprocedure(path);
173 }
174 }
175
176
177
178
179
180
181
182
183 private synchronized void startNewSubprocedure(String path) {
184 LOG.debug("Found procedure znode: " + path);
185 String opName = ZKUtil.getNodeName(path);
186
187 String abortZNode = zkController.getAbortZNode(opName);
188 try {
189 if (ZKUtil.watchAndCheckExists(zkController.getWatcher(), abortZNode)) {
190 LOG.debug("Not starting:" + opName + " because we already have an abort notification.");
191 return;
192 }
193 } catch (KeeperException e) {
194 member.controllerConnectionFailure("Failed to get the abort znode (" + abortZNode
195 + ") for procedure :" + opName, new IOException(e));
196 return;
197 }
198
199
200 Subprocedure subproc = null;
201 try {
202 byte[] data = ZKUtil.getData(zkController.getWatcher(), path);
203 LOG.debug("start proc data length is " + data.length);
204 if (!ProtobufUtil.isPBMagicPrefix(data)) {
205 String msg = "Data in for starting procuedure " + opName +
206 " is illegally formatted (no pb magic). " +
207 "Killing the procedure: " + Bytes.toString(data);
208 LOG.error(msg);
209 throw new IllegalArgumentException(msg);
210 }
211 data = Arrays.copyOfRange(data, ProtobufUtil.lengthOfPBMagic(), data.length);
212 LOG.debug("Found data for znode:" + path);
213 subproc = member.createSubprocedure(opName, data);
214 member.submitSubprocedure(subproc);
215 } catch (IllegalArgumentException iae ) {
216 LOG.error("Illegal argument exception", iae);
217 sendMemberAborted(subproc, new ForeignException(getMemberName(), iae));
218 } catch (IllegalStateException ise) {
219 LOG.error("Illegal state exception ", ise);
220 sendMemberAborted(subproc, new ForeignException(getMemberName(), ise));
221 } catch (KeeperException e) {
222 member.controllerConnectionFailure("Failed to get data for new procedure:" + opName,
223 new IOException(e));
224 }
225 }
226
227
228
229
230
231
232
233 @Override
234 public void sendMemberAcquired(Subprocedure sub) throws IOException {
235 String procName = sub.getName();
236 try {
237 LOG.debug("Member: '" + memberName + "' joining acquired barrier for procedure (" + procName
238 + ") in zk");
239 String acquiredZNode = ZKUtil.joinZNode(ZKProcedureUtil.getAcquireBarrierNode(
240 zkController, procName), memberName);
241 ZKUtil.createAndFailSilent(zkController.getWatcher(), acquiredZNode);
242
243
244 String reachedBarrier = zkController.getReachedBarrierNode(procName);
245 LOG.debug("Watch for global barrier reached:" + reachedBarrier);
246 if (ZKUtil.watchAndCheckExists(zkController.getWatcher(), reachedBarrier)) {
247 receivedReachedGlobalBarrier(reachedBarrier);
248 }
249 } catch (KeeperException e) {
250 member.controllerConnectionFailure("Failed to acquire barrier for procedure: "
251 + procName + " and member: " + memberName, new IOException(e));
252 }
253 }
254
255
256
257
258 @Override
259 public void sendMemberCompleted(Subprocedure sub) throws IOException {
260 String procName = sub.getName();
261 LOG.debug("Marking procedure '" + procName + "' completed for member '" + memberName
262 + "' in zk");
263 String joinPath = ZKUtil.joinZNode(zkController.getReachedBarrierNode(procName), memberName);
264 try {
265 ZKUtil.createAndFailSilent(zkController.getWatcher(), joinPath);
266 } catch (KeeperException e) {
267 member.controllerConnectionFailure("Failed to post zk node:" + joinPath
268 + " to join procedure barrier.", new IOException(e));
269 }
270 }
271
272
273
274
275
276 @Override
277 public void sendMemberAborted(Subprocedure sub, ForeignException ee) {
278 if (sub == null) {
279 LOG.error("Failed due to null subprocedure", ee);
280 return;
281 }
282 String procName = sub.getName();
283 LOG.debug("Aborting procedure (" + procName + ") in zk");
284 String procAbortZNode = zkController.getAbortZNode(procName);
285 try {
286 String source = (ee.getSource() == null) ? memberName: ee.getSource();
287 byte[] errorInfo = ProtobufUtil.prependPBMagic(ForeignException.serialize(source, ee));
288 ZKUtil.createAndFailSilent(zkController.getWatcher(), procAbortZNode, errorInfo);
289 LOG.debug("Finished creating abort znode:" + procAbortZNode);
290 } catch (KeeperException e) {
291
292
293 zkController.logZKTree(zkController.getBaseZnode());
294 member.controllerConnectionFailure("Failed to post zk node:" + procAbortZNode
295 + " to abort procedure", new IOException(e));
296 }
297 }
298
299
300
301
302
303 protected void abort(String abortZNode) {
304 LOG.debug("Aborting procedure member for znode " + abortZNode);
305 String opName = ZKUtil.getNodeName(abortZNode);
306 try {
307 byte[] data = ZKUtil.getData(zkController.getWatcher(), abortZNode);
308
309
310 ForeignException ee;
311 try {
312 if (!ProtobufUtil.isPBMagicPrefix(data)) {
313 String msg = "Illegally formatted data in abort node for proc " + opName
314 + ". Killing the procedure.";
315 LOG.error(msg);
316
317 ee = new ForeignException(getMemberName(), new IllegalArgumentException(msg));
318 } else {
319 data = Arrays.copyOfRange(data, ProtobufUtil.lengthOfPBMagic(), data.length);
320 ee = ForeignException.deserialize(data);
321 }
322 } catch (InvalidProtocolBufferException e) {
323 LOG.warn("Got an error notification for op:" + opName
324 + " but we can't read the information. Killing the procedure.");
325
326 ee = new ForeignException(getMemberName(), e);
327 }
328
329 this.member.receiveAbortProcedure(opName, ee);
330 } catch (KeeperException e) {
331 member.controllerConnectionFailure("Failed to get data for abort znode:" + abortZNode
332 + zkController.getAbortZnode(), new IOException(e));
333 }
334 }
335
336 public void start(final String memberName, final ProcedureMember listener) {
337 LOG.debug("Starting procedure member '" + memberName + "'");
338 this.member = listener;
339 this.memberName = memberName;
340 watchForAbortedProcedures();
341 waitForNewProcedures();
342 }
343
344 @Override
345 public void close() throws IOException {
346 zkController.close();
347 }
348
349 }