1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master.balancer;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.HashMap;
24 import java.util.List;
25 import java.util.Map;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.hbase.classification.InterfaceAudience;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
32 import org.apache.hadoop.hbase.HRegionInfo;
33 import org.apache.hadoop.hbase.NamespaceDescriptor;
34 import org.apache.hadoop.hbase.ServerLoad;
35 import org.apache.hadoop.hbase.ServerName;
36 import org.apache.hadoop.hbase.master.LoadBalancer;
37 import org.apache.hadoop.hbase.master.RackManager;
38 import org.apache.hadoop.hbase.master.RegionPlan;
39 import org.apache.hadoop.hbase.master.ServerManager;
40 import org.apache.hadoop.hbase.master.SnapshotOfRegionAssignmentFromMeta;
41 import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan.Position;
42 import org.apache.hadoop.hbase.util.Pair;
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
58 public class FavoredNodeLoadBalancer extends BaseLoadBalancer {
59 private static final Log LOG = LogFactory.getLog(FavoredNodeLoadBalancer.class);
60
61 private FavoredNodesPlan globalFavoredNodesAssignmentPlan;
62 private RackManager rackManager;
63 Configuration conf;
64
65 @Override
66 public void setConf(Configuration conf) {
67 globalFavoredNodesAssignmentPlan = new FavoredNodesPlan();
68 this.rackManager = new RackManager(conf);
69 this.conf = conf;
70 }
71
72 @Override
73 public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState) {
74
75 List<RegionPlan> plans = new ArrayList<RegionPlan>();
76
77 SnapshotOfRegionAssignmentFromMeta snaphotOfRegionAssignment =
78 new SnapshotOfRegionAssignmentFromMeta(super.services.getCatalogTracker());
79 try {
80 snaphotOfRegionAssignment.initialize();
81 } catch (IOException ie) {
82 LOG.warn("Not running balancer since exception was thrown " + ie);
83 return plans;
84 }
85 globalFavoredNodesAssignmentPlan = snaphotOfRegionAssignment.getExistingAssignmentPlan();
86 Map<ServerName, ServerName> serverNameToServerNameWithoutCode =
87 new HashMap<ServerName, ServerName>();
88 Map<ServerName, ServerName> serverNameWithoutCodeToServerName =
89 new HashMap<ServerName, ServerName>();
90 ServerManager serverMgr = super.services.getServerManager();
91 for (ServerName sn: serverMgr.getOnlineServersList()) {
92 ServerName s = ServerName.valueOf(sn.getHostname(), sn.getPort(), ServerName.NON_STARTCODE);
93 serverNameToServerNameWithoutCode.put(sn, s);
94 serverNameWithoutCodeToServerName.put(s, sn);
95 }
96 for (Map.Entry<ServerName, List<HRegionInfo>> entry : clusterState.entrySet()) {
97 ServerName currentServer = entry.getKey();
98
99 ServerName currentServerWithoutStartCode = ServerName.valueOf(currentServer.getHostname(),
100 currentServer.getPort(), ServerName.NON_STARTCODE);
101 List<HRegionInfo> list = entry.getValue();
102 for (HRegionInfo region : list) {
103 if(region.getTable().getNamespaceAsString()
104 .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
105 continue;
106 }
107 List<ServerName> favoredNodes = globalFavoredNodesAssignmentPlan.getFavoredNodes(region);
108 if (favoredNodes == null || favoredNodes.get(0).equals(currentServerWithoutStartCode)) {
109 continue;
110 }
111 ServerName destination = null;
112
113 destination = serverNameWithoutCodeToServerName.get(favoredNodes.get(0));
114 if (destination == null) {
115
116 if (currentServerWithoutStartCode.equals(favoredNodes.get(1)) ||
117 currentServerWithoutStartCode.equals(favoredNodes.get(2))) {
118 continue;
119 }
120
121
122 ServerLoad l1 = super.services.getServerManager().getLoad(
123 serverNameWithoutCodeToServerName.get(favoredNodes.get(1)));
124 ServerLoad l2 = super.services.getServerManager().getLoad(
125 serverNameWithoutCodeToServerName.get(favoredNodes.get(2)));
126 if (l1 != null && l2 != null) {
127 if (l1.getLoad() > l2.getLoad()) {
128 destination = serverNameWithoutCodeToServerName.get(favoredNodes.get(2));
129 } else {
130 destination = serverNameWithoutCodeToServerName.get(favoredNodes.get(1));
131 }
132 } else if (l1 != null) {
133 destination = serverNameWithoutCodeToServerName.get(favoredNodes.get(1));
134 } else if (l2 != null) {
135 destination = serverNameWithoutCodeToServerName.get(favoredNodes.get(2));
136 }
137 }
138
139 if (destination != null) {
140 RegionPlan plan = new RegionPlan(region, currentServer, destination);
141 plans.add(plan);
142 }
143 }
144 }
145 return plans;
146 }
147
148 @Override
149 public Map<ServerName, List<HRegionInfo>> roundRobinAssignment(List<HRegionInfo> regions,
150 List<ServerName> servers) {
151 Map<ServerName, List<HRegionInfo>> assignmentMap;
152 try {
153 FavoredNodeAssignmentHelper assignmentHelper =
154 new FavoredNodeAssignmentHelper(servers, rackManager);
155 assignmentHelper.initialize();
156 if (!assignmentHelper.canPlaceFavoredNodes()) {
157 return super.roundRobinAssignment(regions, servers);
158 }
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175 Pair<Map<ServerName,List<HRegionInfo>>, List<HRegionInfo>> segregatedRegions =
176 segregateRegionsAndAssignRegionsWithFavoredNodes(regions, servers);
177 Map<ServerName,List<HRegionInfo>> regionsWithFavoredNodesMap = segregatedRegions.getFirst();
178 List<HRegionInfo> regionsWithNoFavoredNodes = segregatedRegions.getSecond();
179 assignmentMap = new HashMap<ServerName, List<HRegionInfo>>();
180 roundRobinAssignmentImpl(assignmentHelper, assignmentMap, regionsWithNoFavoredNodes,
181 servers);
182
183 assignmentMap.putAll(regionsWithFavoredNodesMap);
184 } catch (Exception ex) {
185 LOG.warn("Encountered exception while doing favored-nodes assignment " + ex +
186 " Falling back to regular assignment");
187 assignmentMap = super.roundRobinAssignment(regions, servers);
188 }
189 return assignmentMap;
190 }
191
192 @Override
193 public ServerName randomAssignment(HRegionInfo regionInfo, List<ServerName> servers) {
194 try {
195 FavoredNodeAssignmentHelper assignmentHelper =
196 new FavoredNodeAssignmentHelper(servers, rackManager);
197 assignmentHelper.initialize();
198 ServerName primary = super.randomAssignment(regionInfo, servers);
199 if (!assignmentHelper.canPlaceFavoredNodes()) {
200 return primary;
201 }
202 List<ServerName> favoredNodes = globalFavoredNodesAssignmentPlan.getFavoredNodes(regionInfo);
203
204
205
206 if (favoredNodes != null) {
207 for (ServerName s : favoredNodes) {
208 ServerName serverWithLegitStartCode = availableServersContains(servers, s);
209 if (serverWithLegitStartCode != null) {
210 return serverWithLegitStartCode;
211 }
212 }
213 }
214 List<HRegionInfo> regions = new ArrayList<HRegionInfo>(1);
215 regions.add(regionInfo);
216 Map<HRegionInfo, ServerName> primaryRSMap = new HashMap<HRegionInfo, ServerName>(1);
217 primaryRSMap.put(regionInfo, primary);
218 assignSecondaryAndTertiaryNodesForRegion(assignmentHelper, regions, primaryRSMap);
219 return primary;
220 } catch (Exception ex) {
221 LOG.warn("Encountered exception while doing favored-nodes (random)assignment " + ex +
222 " Falling back to regular assignment");
223 return super.randomAssignment(regionInfo, servers);
224 }
225 }
226
227 private Pair<Map<ServerName, List<HRegionInfo>>, List<HRegionInfo>>
228 segregateRegionsAndAssignRegionsWithFavoredNodes(List<HRegionInfo> regions,
229 List<ServerName> availableServers) {
230 Map<ServerName, List<HRegionInfo>> assignmentMapForFavoredNodes =
231 new HashMap<ServerName, List<HRegionInfo>>(regions.size() / 2);
232 List<HRegionInfo> regionsWithNoFavoredNodes = new ArrayList<HRegionInfo>(regions.size()/2);
233 for (HRegionInfo region : regions) {
234 List<ServerName> favoredNodes = globalFavoredNodesAssignmentPlan.getFavoredNodes(region);
235 ServerName primaryHost = null;
236 ServerName secondaryHost = null;
237 ServerName tertiaryHost = null;
238 if (favoredNodes != null) {
239 for (ServerName s : favoredNodes) {
240 ServerName serverWithLegitStartCode = availableServersContains(availableServers, s);
241 if (serverWithLegitStartCode != null) {
242 FavoredNodesPlan.Position position =
243 FavoredNodesPlan.getFavoredServerPosition(favoredNodes, s);
244 if (Position.PRIMARY.equals(position)) {
245 primaryHost = serverWithLegitStartCode;
246 } else if (Position.SECONDARY.equals(position)) {
247 secondaryHost = serverWithLegitStartCode;
248 } else if (Position.TERTIARY.equals(position)) {
249 tertiaryHost = serverWithLegitStartCode;
250 }
251 }
252 }
253 assignRegionToAvailableFavoredNode(assignmentMapForFavoredNodes, region,
254 primaryHost, secondaryHost, tertiaryHost);
255 }
256 if (primaryHost == null && secondaryHost == null && tertiaryHost == null) {
257
258 regionsWithNoFavoredNodes.add(region);
259 }
260 }
261 return new Pair<Map<ServerName, List<HRegionInfo>>, List<HRegionInfo>>(
262 assignmentMapForFavoredNodes, regionsWithNoFavoredNodes);
263 }
264
265
266
267
268 private ServerName availableServersContains(List<ServerName> servers, ServerName favoredNode) {
269 for (ServerName server : servers) {
270 if (ServerName.isSameHostnameAndPort(favoredNode, server)) {
271 return server;
272 }
273 }
274 return null;
275 }
276
277 private void assignRegionToAvailableFavoredNode(Map<ServerName,
278 List<HRegionInfo>> assignmentMapForFavoredNodes, HRegionInfo region, ServerName primaryHost,
279 ServerName secondaryHost, ServerName tertiaryHost) {
280 if (primaryHost != null) {
281 addRegionToMap(assignmentMapForFavoredNodes, region, primaryHost);
282 } else if (secondaryHost != null && tertiaryHost != null) {
283
284
285 ServerName s;
286 ServerLoad tertiaryLoad = super.services.getServerManager().getLoad(tertiaryHost);
287 ServerLoad secondaryLoad = super.services.getServerManager().getLoad(secondaryHost);
288 if (secondaryLoad.getLoad() < tertiaryLoad.getLoad()) {
289 s = secondaryHost;
290 } else {
291 s = tertiaryHost;
292 }
293 addRegionToMap(assignmentMapForFavoredNodes, region, s);
294 } else if (secondaryHost != null) {
295 addRegionToMap(assignmentMapForFavoredNodes, region, secondaryHost);
296 } else if (tertiaryHost != null) {
297 addRegionToMap(assignmentMapForFavoredNodes, region, tertiaryHost);
298 }
299 }
300
301 private void addRegionToMap(Map<ServerName, List<HRegionInfo>> assignmentMapForFavoredNodes,
302 HRegionInfo region, ServerName host) {
303 List<HRegionInfo> regionsOnServer = null;
304 if ((regionsOnServer = assignmentMapForFavoredNodes.get(host)) == null) {
305 regionsOnServer = new ArrayList<HRegionInfo>();
306 assignmentMapForFavoredNodes.put(host, regionsOnServer);
307 }
308 regionsOnServer.add(region);
309 }
310
311 public List<ServerName> getFavoredNodes(HRegionInfo regionInfo) {
312 return this.globalFavoredNodesAssignmentPlan.getFavoredNodes(regionInfo);
313 }
314
315 private void roundRobinAssignmentImpl(FavoredNodeAssignmentHelper assignmentHelper,
316 Map<ServerName, List<HRegionInfo>> assignmentMap,
317 List<HRegionInfo> regions, List<ServerName> servers) {
318 Map<HRegionInfo, ServerName> primaryRSMap = new HashMap<HRegionInfo, ServerName>();
319
320 assignmentHelper.placePrimaryRSAsRoundRobin(assignmentMap, primaryRSMap, regions);
321 assignSecondaryAndTertiaryNodesForRegion(assignmentHelper, regions, primaryRSMap);
322 }
323
324 private void assignSecondaryAndTertiaryNodesForRegion(
325 FavoredNodeAssignmentHelper assignmentHelper,
326 List<HRegionInfo> regions, Map<HRegionInfo, ServerName> primaryRSMap) {
327
328 Map<HRegionInfo, ServerName[]> secondaryAndTertiaryRSMap =
329 assignmentHelper.placeSecondaryAndTertiaryRS(primaryRSMap);
330
331 for (HRegionInfo region : regions) {
332
333
334 List<ServerName> favoredNodesForRegion = new ArrayList<ServerName>(3);
335 ServerName sn = primaryRSMap.get(region);
336 favoredNodesForRegion.add(ServerName.valueOf(sn.getHostname(), sn.getPort(),
337 ServerName.NON_STARTCODE));
338 ServerName[] secondaryAndTertiaryNodes = secondaryAndTertiaryRSMap.get(region);
339 if (secondaryAndTertiaryNodes != null) {
340 favoredNodesForRegion.add(ServerName.valueOf(secondaryAndTertiaryNodes[0].getHostname(),
341 secondaryAndTertiaryNodes[0].getPort(), ServerName.NON_STARTCODE));
342 favoredNodesForRegion.add(ServerName.valueOf(secondaryAndTertiaryNodes[1].getHostname(),
343 secondaryAndTertiaryNodes[1].getPort(), ServerName.NON_STARTCODE));
344 }
345 globalFavoredNodesAssignmentPlan.updateFavoredNodesMap(region, favoredNodesForRegion);
346 }
347 }
348 }