1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.fs;
22
23 import java.io.Closeable;
24 import java.io.IOException;
25 import java.lang.reflect.Field;
26 import java.lang.reflect.InvocationHandler;
27 import java.lang.reflect.InvocationTargetException;
28 import java.lang.reflect.Method;
29 import java.lang.reflect.Modifier;
30 import java.lang.reflect.Proxy;
31 import java.lang.reflect.UndeclaredThrowableException;
32 import java.net.URI;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataOutputStream;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.FilterFileSystem;
40 import org.apache.hadoop.fs.LocalFileSystem;
41 import org.apache.hadoop.fs.Path;
42 import org.apache.hadoop.hbase.ServerName;
43 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
44 import org.apache.hadoop.hdfs.DFSClient;
45 import org.apache.hadoop.hdfs.DistributedFileSystem;
46 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
47 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
48 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
49 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
50 import org.apache.hadoop.ipc.RPC;
51 import org.apache.hadoop.util.Progressable;
52 import org.apache.hadoop.util.ReflectionUtils;
53
54
55
56
57
58
59
60
61 public class HFileSystem extends FilterFileSystem {
62 public static final Log LOG = LogFactory.getLog(HFileSystem.class);
63
64 private final FileSystem noChecksumFs;
65 private final boolean useHBaseChecksum;
66
67
68
69
70
71
72
73
74 public HFileSystem(Configuration conf, boolean useHBaseChecksum)
75 throws IOException {
76
77
78
79
80 this.fs = FileSystem.get(conf);
81 this.useHBaseChecksum = useHBaseChecksum;
82
83 fs.initialize(getDefaultUri(conf), conf);
84
85
86 if (fs instanceof LocalFileSystem) {
87 setWriteChecksum(fs, false);
88 setVerifyChecksum(fs, false);
89 }
90
91 addLocationsOrderInterceptor(conf);
92
93
94
95
96
97
98
99
100
101 if (useHBaseChecksum && !(fs instanceof LocalFileSystem)) {
102 conf = new Configuration(conf);
103 conf.setBoolean("dfs.client.read.shortcircuit.skip.checksum", true);
104 this.noChecksumFs = newInstanceFileSystem(conf);
105 this.noChecksumFs.setVerifyChecksum(false);
106 } else {
107 this.noChecksumFs = fs;
108 }
109 }
110
111 private static void setWriteChecksum(FileSystem fs, boolean writeChecksum) throws IOException {
112
113 try {
114 FileSystem.class.getMethod("setWriteChecksum", boolean.class).invoke(fs, writeChecksum);
115 } catch (NoSuchMethodException e) {
116 LOG.debug("FileSystem does not support setWriteChecksum, ignoring");
117 } catch (Throwable t) {
118 throw new IOException("setWriteChecksum failed", t);
119 }
120 }
121
122 private static void setVerifyChecksum(FileSystem fs, boolean verifyChecksum) throws IOException {
123 fs.setVerifyChecksum(verifyChecksum);
124 }
125
126
127
128
129
130
131
132 public HFileSystem(FileSystem fs) {
133 this.fs = fs;
134 this.noChecksumFs = fs;
135 this.useHBaseChecksum = false;
136 }
137
138
139
140
141
142
143
144
145 public FileSystem getNoChecksumFs() {
146 return noChecksumFs;
147 }
148
149
150
151
152
153 public FileSystem getBackingFs() throws IOException {
154 return fs;
155 }
156
157
158
159
160
161
162 public boolean useHBaseChecksum() {
163 return useHBaseChecksum;
164 }
165
166
167
168
169 @Override
170 public void close() throws IOException {
171 super.close();
172 if (this.noChecksumFs != fs) {
173 this.noChecksumFs.close();
174 }
175 }
176
177
178
179
180
181
182
183
184
185 private static FileSystem newInstanceFileSystem(Configuration conf)
186 throws IOException {
187 URI uri = FileSystem.getDefaultUri(conf);
188 FileSystem fs = null;
189 Class<?> clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null);
190 if (clazz != null) {
191
192 fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
193 fs.initialize(uri, conf);
194 } else {
195
196
197
198 Configuration clone = new Configuration(conf);
199 clone.setBoolean("fs." + uri.getScheme() + ".impl.disable.cache", true);
200 fs = FileSystem.get(uri, clone);
201 }
202 if (fs == null) {
203 throw new IOException("No FileSystem for scheme: " + uri.getScheme());
204 }
205
206 return fs;
207 }
208
209 public static boolean addLocationsOrderInterceptor(Configuration conf) throws IOException {
210 return addLocationsOrderInterceptor(conf, new ReorderWALBlocks());
211 }
212
213
214
215
216
217
218
219
220
221 static boolean addLocationsOrderInterceptor(Configuration conf, final ReorderBlocks lrb) {
222 if (!conf.getBoolean("hbase.filesystem.reorder.blocks", true)) {
223 LOG.debug("addLocationsOrderInterceptor configured to false");
224 return false;
225 }
226
227 FileSystem fs;
228 try {
229 fs = FileSystem.get(conf);
230 } catch (IOException e) {
231 LOG.warn("Can't get the file system from the conf.", e);
232 return false;
233 }
234
235 if (!(fs instanceof DistributedFileSystem)) {
236 LOG.debug("The file system is not a DistributedFileSystem. " +
237 "Skipping on block location reordering");
238 return false;
239 }
240
241 DistributedFileSystem dfs = (DistributedFileSystem) fs;
242 DFSClient dfsc = dfs.getClient();
243 if (dfsc == null) {
244 LOG.warn("The DistributedFileSystem does not contain a DFSClient. Can't add the location " +
245 "block reordering interceptor. Continuing, but this is unexpected."
246 );
247 return false;
248 }
249
250 try {
251 Field nf = DFSClient.class.getDeclaredField("namenode");
252 nf.setAccessible(true);
253 Field modifiersField = Field.class.getDeclaredField("modifiers");
254 modifiersField.setAccessible(true);
255 modifiersField.setInt(nf, nf.getModifiers() & ~Modifier.FINAL);
256
257 ClientProtocol namenode = (ClientProtocol) nf.get(dfsc);
258 if (namenode == null) {
259 LOG.warn("The DFSClient is not linked to a namenode. Can't add the location block" +
260 " reordering interceptor. Continuing, but this is unexpected."
261 );
262 return false;
263 }
264
265 ClientProtocol cp1 = createReorderingProxy(namenode, lrb, conf);
266 nf.set(dfsc, cp1);
267 LOG.info("Added intercepting call to namenode#getBlockLocations so can do block reordering" +
268 " using class " + lrb.getClass());
269 } catch (NoSuchFieldException e) {
270 LOG.warn("Can't modify the DFSClient#namenode field to add the location reorder.", e);
271 return false;
272 } catch (IllegalAccessException e) {
273 LOG.warn("Can't modify the DFSClient#namenode field to add the location reorder.", e);
274 return false;
275 }
276
277 return true;
278 }
279
280 private static ClientProtocol createReorderingProxy(final ClientProtocol cp,
281 final ReorderBlocks lrb, final Configuration conf) {
282 return (ClientProtocol) Proxy.newProxyInstance
283 (cp.getClass().getClassLoader(),
284 new Class[]{ClientProtocol.class, Closeable.class},
285 new InvocationHandler() {
286 public Object invoke(Object proxy, Method method,
287 Object[] args) throws Throwable {
288 try {
289 if ((args == null || args.length == 0)
290 && "close".equals(method.getName())) {
291 RPC.stopProxy(cp);
292 return null;
293 } else {
294 Object res = method.invoke(cp, args);
295 if (res != null && args != null && args.length == 3
296 && "getBlockLocations".equals(method.getName())
297 && res instanceof LocatedBlocks
298 && args[0] instanceof String
299 && args[0] != null) {
300 lrb.reorderBlocks(conf, (LocatedBlocks) res, (String) args[0]);
301 }
302 return res;
303 }
304 } catch (InvocationTargetException ite) {
305
306
307 Throwable cause = ite.getCause();
308 if (cause == null){
309 throw new RuntimeException(
310 "Proxy invocation failed and getCause is null", ite);
311 }
312 if (cause instanceof UndeclaredThrowableException) {
313 Throwable causeCause = cause.getCause();
314 if (causeCause == null) {
315 throw new RuntimeException("UndeclaredThrowableException had null cause!");
316 }
317 cause = cause.getCause();
318 }
319 throw cause;
320 }
321 }
322 });
323 }
324
325
326
327
328 interface ReorderBlocks {
329
330
331
332
333
334
335
336 void reorderBlocks(Configuration conf, LocatedBlocks lbs, String src) throws IOException;
337 }
338
339
340
341
342
343
344 static class ReorderWALBlocks implements ReorderBlocks {
345 public void reorderBlocks(Configuration conf, LocatedBlocks lbs, String src)
346 throws IOException {
347
348 ServerName sn = HLogUtil.getServerNameFromHLogDirectoryName(conf, src);
349 if (sn == null) {
350
351 return;
352 }
353
354
355 String hostName = sn.getHostname();
356 if (LOG.isTraceEnabled()) {
357 LOG.trace(src +
358 " is an HLog file, so reordering blocks, last hostname will be:" + hostName);
359 }
360
361
362 for (LocatedBlock lb : lbs.getLocatedBlocks()) {
363 DatanodeInfo[] dnis = lb.getLocations();
364 if (dnis != null && dnis.length > 1) {
365 boolean found = false;
366 for (int i = 0; i < dnis.length - 1 && !found; i++) {
367 if (hostName.equals(dnis[i].getHostName())) {
368
369 DatanodeInfo toLast = dnis[i];
370 System.arraycopy(dnis, i + 1, dnis, i, dnis.length - i - 1);
371 dnis[dnis.length - 1] = toLast;
372 found = true;
373 }
374 }
375 }
376 }
377 }
378 }
379
380
381
382
383
384
385
386 static public FileSystem get(Configuration conf) throws IOException {
387 return new HFileSystem(conf, true);
388 }
389
390
391
392
393 static public FileSystem getLocalFs(Configuration conf) throws IOException {
394 return new HFileSystem(FileSystem.getLocal(conf));
395 }
396
397
398
399
400
401
402 @SuppressWarnings("deprecation")
403 public FSDataOutputStream createNonRecursive(Path f,
404 boolean overwrite,
405 int bufferSize, short replication, long blockSize,
406 Progressable progress) throws IOException {
407 return fs.createNonRecursive(f, overwrite, bufferSize, replication,
408 blockSize, progress);
409 }
410 }