1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import java.io.IOException;
22 import java.util.HashMap;
23 import java.util.Map;
24
25 import java.util.concurrent.atomic.AtomicInteger;
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.hbase.classification.InterfaceAudience;
29 import org.apache.hadoop.fs.BlockLocation;
30 import org.apache.hadoop.fs.FileStatus;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.HConstants;
34 import org.apache.hadoop.hbase.util.FSUtils;
35
36
37
38
39
40
41 @InterfaceAudience.Private
42 class FSRegionScanner implements Runnable {
43 static private final Log LOG = LogFactory.getLog(FSRegionScanner.class);
44
45 private Path regionPath;
46
47
48
49
50 private FileSystem fs;
51
52
53
54
55 private Map<String,String> regionToBestLocalityRSMapping;
56
57
58
59
60
61 private Map<String, Map<String, Float>> regionDegreeLocalityMapping;
62
63 FSRegionScanner(FileSystem fs, Path regionPath,
64 Map<String, String> regionToBestLocalityRSMapping,
65 Map<String, Map<String, Float>> regionDegreeLocalityMapping) {
66 this.fs = fs;
67 this.regionPath = regionPath;
68 this.regionToBestLocalityRSMapping = regionToBestLocalityRSMapping;
69 this.regionDegreeLocalityMapping = regionDegreeLocalityMapping;
70 }
71
72 @Override
73 public void run() {
74 try {
75
76 Map<String, AtomicInteger> blockCountMap = new HashMap<String, AtomicInteger>();
77
78
79 String tableName = regionPath.getParent().getName();
80 int totalBlkCount = 0;
81
82
83 FileStatus[] cfList = fs.listStatus(regionPath, new FSUtils.FamilyDirFilter(fs));
84 if (null == cfList) {
85 return;
86 }
87
88
89 for (FileStatus cfStatus : cfList) {
90 if (!cfStatus.isDir()) {
91
92 continue;
93 }
94
95 FileStatus[] storeFileLists = fs.listStatus(cfStatus.getPath());
96 if (null == storeFileLists) {
97 continue;
98 }
99
100 for (FileStatus storeFile : storeFileLists) {
101 BlockLocation[] blkLocations =
102 fs.getFileBlockLocations(storeFile, 0, storeFile.getLen());
103 if (null == blkLocations) {
104 continue;
105 }
106
107 totalBlkCount += blkLocations.length;
108 for(BlockLocation blk: blkLocations) {
109 for (String host: blk.getHosts()) {
110 AtomicInteger count = blockCountMap.get(host);
111 if (count == null) {
112 count = new AtomicInteger(0);
113 blockCountMap.put(host, count);
114 }
115 count.incrementAndGet();
116 }
117 }
118 }
119 }
120
121 if (regionToBestLocalityRSMapping != null) {
122 int largestBlkCount = 0;
123 String hostToRun = null;
124 for (Map.Entry<String, AtomicInteger> entry : blockCountMap.entrySet()) {
125 String host = entry.getKey();
126
127 int tmp = entry.getValue().get();
128 if (tmp > largestBlkCount) {
129 largestBlkCount = tmp;
130 hostToRun = host;
131 }
132 }
133
134
135 if (null == hostToRun) {
136 return;
137 }
138
139 if (hostToRun.endsWith(".")) {
140 hostToRun = hostToRun.substring(0, hostToRun.length()-1);
141 }
142 String name = tableName + ":" + regionPath.getName();
143 synchronized (regionToBestLocalityRSMapping) {
144 regionToBestLocalityRSMapping.put(name, hostToRun);
145 }
146 }
147
148 if (regionDegreeLocalityMapping != null && totalBlkCount > 0) {
149 Map<String, Float> hostLocalityMap = new HashMap<String, Float>();
150 for (Map.Entry<String, AtomicInteger> entry : blockCountMap.entrySet()) {
151 String host = entry.getKey();
152 if (host.endsWith(".")) {
153 host = host.substring(0, host.length() - 1);
154 }
155
156 float locality = ((float)entry.getValue().get()) / totalBlkCount;
157 hostLocalityMap.put(host, locality);
158 }
159
160
161 regionDegreeLocalityMapping.put(regionPath.getName(), hostLocalityMap);
162 }
163 } catch (IOException e) {
164 LOG.warn("Problem scanning file system", e);
165 } catch (RuntimeException e) {
166 LOG.warn("Problem scanning file system", e);
167 }
168 }
169 }