1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.lang.reflect.Method;
25 import java.net.InetSocketAddress;
26 import java.net.URI;
27 import java.util.HashSet;
28 import java.util.Map;
29 import java.util.Set;
30
31 import com.google.common.collect.Sets;
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.hbase.classification.InterfaceAudience;
35 import org.apache.hadoop.hbase.classification.InterfaceStability;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileSystem;
38 import org.apache.hadoop.fs.Path;
39 import org.apache.hadoop.hdfs.DistributedFileSystem;
40 import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
41
42
43
44
45
46 @InterfaceAudience.Private
47 @InterfaceStability.Evolving
48 public class FSHDFSUtils extends FSUtils {
49 private static final Log LOG = LogFactory.getLog(FSHDFSUtils.class);
50 private static Class dfsUtilClazz;
51 private static Method getNNAddressesMethod;
52
53
54
55
56
57
58 private static Set<InetSocketAddress> getNNAddresses(DistributedFileSystem fs,
59 Configuration conf) {
60 Set<InetSocketAddress> addresses = new HashSet<InetSocketAddress>();
61 String serviceName = fs.getCanonicalServiceName();
62
63 if (serviceName.startsWith("ha-hdfs")) {
64 try {
65 if (dfsUtilClazz == null) {
66 dfsUtilClazz = Class.forName("org.apache.hadoop.hdfs.DFSUtil");
67 }
68 if (getNNAddressesMethod == null) {
69 getNNAddressesMethod =
70 dfsUtilClazz.getMethod("getNNServiceRpcAddresses", Configuration.class);
71 }
72
73 Map<String, Map<String, InetSocketAddress>> addressMap =
74 (Map<String, Map<String, InetSocketAddress>>) getNNAddressesMethod
75 .invoke(null, conf);
76 for (Map.Entry<String, Map<String, InetSocketAddress>> entry : addressMap.entrySet()) {
77 Map<String, InetSocketAddress> nnMap = entry.getValue();
78 for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) {
79 InetSocketAddress addr = e2.getValue();
80 addresses.add(addr);
81 }
82 }
83 } catch (Exception e) {
84 LOG.warn("DFSUtil.getNNServiceRpcAddresses failed. serviceName=" + serviceName, e);
85 }
86 } else {
87 URI uri = fs.getUri();
88 int port = uri.getPort();
89 if (port < 0) {
90 int idx = serviceName.indexOf(':');
91 port = Integer.parseInt(serviceName.substring(idx+1));
92 }
93 InetSocketAddress addr = new InetSocketAddress(uri.getHost(), port);
94 addresses.add(addr);
95 }
96
97 return addresses;
98 }
99
100
101
102
103
104
105
106 public static boolean isSameHdfs(Configuration conf, FileSystem srcFs, FileSystem desFs) {
107
108
109 String srcServiceName = srcFs.getCanonicalServiceName();
110 String desServiceName = desFs.getCanonicalServiceName();
111
112 if (srcServiceName == null || desServiceName == null) {
113 return false;
114 }
115 if (srcServiceName.equals(desServiceName)) {
116 return true;
117 }
118 if (srcFs instanceof DistributedFileSystem && desFs instanceof DistributedFileSystem) {
119
120
121
122 Set<InetSocketAddress> srcAddrs = getNNAddresses((DistributedFileSystem) srcFs, conf);
123 Set<InetSocketAddress> desAddrs = getNNAddresses((DistributedFileSystem) desFs, conf);
124 if (Sets.intersection(srcAddrs, desAddrs).size() > 0) {
125 return true;
126 }
127 }
128
129 return false;
130 }
131
132
133
134
135 @Override
136 public void recoverFileLease(final FileSystem fs, final Path p,
137 Configuration conf, CancelableProgressable reporter)
138 throws IOException {
139
140 if (!(fs instanceof DistributedFileSystem)) return;
141 recoverDFSFileLease((DistributedFileSystem)fs, p, conf, reporter);
142 }
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170 boolean recoverDFSFileLease(final DistributedFileSystem dfs, final Path p,
171 final Configuration conf, final CancelableProgressable reporter)
172 throws IOException {
173 LOG.info("Recovering lease on dfs file " + p);
174 long startWaiting = EnvironmentEdgeManager.currentTimeMillis();
175
176
177
178 long recoveryTimeout = conf.getInt("hbase.lease.recovery.timeout", 900000) + startWaiting;
179
180 long firstPause = conf.getInt("hbase.lease.recovery.first.pause", 4000);
181
182
183
184 long subsequentPause = conf.getInt("hbase.lease.recovery.dfs.timeout", 61 * 1000);
185
186 Method isFileClosedMeth = null;
187
188 boolean findIsFileClosedMeth = true;
189 boolean recovered = false;
190
191 for (int nbAttempt = 0; !recovered; nbAttempt++) {
192 recovered = recoverLease(dfs, nbAttempt, p, startWaiting);
193 if (recovered) break;
194 checkIfCancelled(reporter);
195 if (checkIfTimedout(conf, recoveryTimeout, nbAttempt, p, startWaiting)) break;
196 try {
197
198 if (nbAttempt == 0) {
199 Thread.sleep(firstPause);
200 } else {
201
202
203 long localStartWaiting = EnvironmentEdgeManager.currentTimeMillis();
204 while ((EnvironmentEdgeManager.currentTimeMillis() - localStartWaiting) <
205 subsequentPause) {
206 Thread.sleep(conf.getInt("hbase.lease.recovery.pause", 1000));
207 if (findIsFileClosedMeth) {
208 try {
209 isFileClosedMeth = dfs.getClass().getMethod("isFileClosed",
210 new Class[]{ Path.class });
211 } catch (NoSuchMethodException nsme) {
212 LOG.debug("isFileClosed not available");
213 } finally {
214 findIsFileClosedMeth = false;
215 }
216 }
217 if (isFileClosedMeth != null && isFileClosed(dfs, isFileClosedMeth, p)) {
218 recovered = true;
219 break;
220 }
221 checkIfCancelled(reporter);
222 }
223 }
224 } catch (InterruptedException ie) {
225 InterruptedIOException iioe = new InterruptedIOException();
226 iioe.initCause(ie);
227 throw iioe;
228 }
229 }
230 return recovered;
231 }
232
233 boolean checkIfTimedout(final Configuration conf, final long recoveryTimeout,
234 final int nbAttempt, final Path p, final long startWaiting) {
235 if (recoveryTimeout < EnvironmentEdgeManager.currentTimeMillis()) {
236 LOG.warn("Cannot recoverLease after trying for " +
237 conf.getInt("hbase.lease.recovery.timeout", 900000) +
238 "ms (hbase.lease.recovery.timeout); continuing, but may be DATALOSS!!!; " +
239 getLogMessageDetail(nbAttempt, p, startWaiting));
240 return true;
241 }
242 return false;
243 }
244
245
246
247
248
249
250
251
252
253
254 boolean recoverLease(final DistributedFileSystem dfs, final int nbAttempt, final Path p,
255 final long startWaiting)
256 throws FileNotFoundException {
257 boolean recovered = false;
258 try {
259 recovered = dfs.recoverLease(p);
260 LOG.info("recoverLease=" + recovered + ", " +
261 getLogMessageDetail(nbAttempt, p, startWaiting));
262 } catch (IOException e) {
263 if (e instanceof LeaseExpiredException && e.getMessage().contains("File does not exist")) {
264
265 throw new FileNotFoundException("The given HLog wasn't found at " + p);
266 } else if (e instanceof FileNotFoundException) {
267 throw (FileNotFoundException)e;
268 }
269 LOG.warn(getLogMessageDetail(nbAttempt, p, startWaiting), e);
270 }
271 return recovered;
272 }
273
274
275
276
277
278
279
280 private String getLogMessageDetail(final int nbAttempt, final Path p, final long startWaiting) {
281 return "attempt=" + nbAttempt + " on file=" + p + " after " +
282 (EnvironmentEdgeManager.currentTimeMillis() - startWaiting) + "ms";
283 }
284
285
286
287
288
289
290
291
292 private boolean isFileClosed(final DistributedFileSystem dfs, final Method m, final Path p) {
293 try {
294 return (Boolean) m.invoke(dfs, p);
295 } catch (SecurityException e) {
296 LOG.warn("No access", e);
297 } catch (Exception e) {
298 LOG.warn("Failed invocation for " + p.toString(), e);
299 }
300 return false;
301 }
302
303 void checkIfCancelled(final CancelableProgressable reporter)
304 throws InterruptedIOException {
305 if (reporter == null) return;
306 if (!reporter.progress()) throw new InterruptedIOException("Operation cancelled");
307 }
308 }