1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.balancer;
19
20 import java.util.ArrayList;
21 import java.util.Arrays;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.Queue;
25 import java.util.TreeMap;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.hbase.ClusterStatus;
31 import org.apache.hadoop.hbase.HBaseConfiguration;
32 import org.apache.hadoop.hbase.HRegionInfo;
33 import org.apache.hadoop.hbase.testclassification.MediumTests;
34 import org.apache.hadoop.hbase.RegionLoad;
35 import org.apache.hadoop.hbase.ServerLoad;
36 import org.apache.hadoop.hbase.ServerName;
37 import org.apache.hadoop.hbase.master.RegionPlan;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.junit.BeforeClass;
40 import org.junit.Test;
41 import org.junit.experimental.categories.Category;
42
43 import static org.junit.Assert.assertEquals;
44 import static org.junit.Assert.assertNotNull;
45 import static org.junit.Assert.assertNull;
46 import static org.junit.Assert.assertTrue;
47 import static org.mockito.Mockito.mock;
48 import static org.mockito.Mockito.when;
49
50 @Category(MediumTests.class)
51 public class TestStochasticLoadBalancer extends BalancerTestBase {
52 public static final String REGION_KEY = "testRegion";
53 private static StochasticLoadBalancer loadBalancer;
54 private static final Log LOG = LogFactory.getLog(TestStochasticLoadBalancer.class);
55
56 @BeforeClass
57 public static void beforeAllTests() throws Exception {
58 Configuration conf = HBaseConfiguration.create();
59 conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f);
60 conf.setFloat("hbase.regions.slop", 0.0f);
61 loadBalancer = new StochasticLoadBalancer();
62 loadBalancer.setConf(conf);
63 }
64
65 int[] largeCluster = new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
66 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
67 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
68 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
69 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
70 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
71 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
72 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
73 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
74 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
75 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
76 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
77 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
78 0, 0, 0, 0, 0, 56 };
79
80
81 int[][] clusterStateMocks = new int[][]{
82
83 new int[]{0},
84 new int[]{1},
85 new int[]{10},
86
87 new int[]{0, 0},
88 new int[]{2, 0},
89 new int[]{2, 1},
90 new int[]{2, 2},
91 new int[]{2, 3},
92 new int[]{2, 4},
93 new int[]{1, 1},
94 new int[]{0, 1},
95 new int[]{10, 1},
96 new int[]{514, 1432},
97 new int[]{48, 53},
98
99 new int[]{0, 1, 2},
100 new int[]{1, 2, 3},
101 new int[]{0, 2, 2},
102 new int[]{0, 3, 0},
103 new int[]{0, 4, 0},
104 new int[]{20, 20, 0},
105
106 new int[]{0, 1, 2, 3},
107 new int[]{4, 0, 0, 0},
108 new int[]{5, 0, 0, 0},
109 new int[]{6, 6, 0, 0},
110 new int[]{6, 2, 0, 0},
111 new int[]{6, 1, 0, 0},
112 new int[]{6, 0, 0, 0},
113 new int[]{4, 4, 4, 7},
114 new int[]{4, 4, 4, 8},
115 new int[]{0, 0, 0, 7},
116
117 new int[]{1, 1, 1, 1, 4},
118
119 new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
120 new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 10},
121 new int[]{6, 6, 5, 6, 6, 6, 6, 6, 6, 1},
122 new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 54},
123 new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 55},
124 new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 56},
125 new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 16},
126 new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 8},
127 new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 9},
128 new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 10},
129 new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 123},
130 new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 155},
131 new int[]{10, 7, 12, 8, 11, 10, 9, 14},
132 new int[]{13, 14, 6, 10, 10, 10, 8, 10},
133 new int[]{130, 14, 60, 10, 100, 10, 80, 10},
134 new int[]{130, 140, 60, 100, 100, 100, 80, 100},
135 largeCluster,
136
137 };
138
139 @Test
140 public void testKeepRegionLoad() throws Exception {
141
142 ServerName sn = ServerName.valueOf("test:8080", 100);
143 int numClusterStatusToAdd = 20000;
144 for (int i = 0; i < numClusterStatusToAdd; i++) {
145 ServerLoad sl = mock(ServerLoad.class);
146
147 RegionLoad rl = mock(RegionLoad.class);
148 when(rl.getStores()).thenReturn(i);
149
150 Map<byte[], RegionLoad> regionLoadMap =
151 new TreeMap<byte[], RegionLoad>(Bytes.BYTES_COMPARATOR);
152 regionLoadMap.put(Bytes.toBytes(REGION_KEY), rl);
153 when(sl.getRegionsLoad()).thenReturn(regionLoadMap);
154
155 ClusterStatus clusterStatus = mock(ClusterStatus.class);
156 when(clusterStatus.getServers()).thenReturn(Arrays.asList(sn));
157 when(clusterStatus.getLoad(sn)).thenReturn(sl);
158
159 loadBalancer.setClusterStatus(clusterStatus);
160 }
161 assertTrue(loadBalancer.loads.get(REGION_KEY) != null);
162 assertTrue(loadBalancer.loads.get(REGION_KEY).size() == 15);
163
164 Queue<RegionLoad> loads = loadBalancer.loads.get(REGION_KEY);
165 int i = 0;
166 while(loads.size() > 0) {
167 RegionLoad rl = loads.remove();
168 assertEquals(i + (numClusterStatusToAdd - 15), rl.getStores());
169 i ++;
170 }
171 }
172
173
174
175
176
177
178
179
180
181 @Test
182 public void testBalanceCluster() throws Exception {
183
184 for (int[] mockCluster : clusterStateMocks) {
185 Map<ServerName, List<HRegionInfo>> servers = mockClusterServers(mockCluster);
186 List<ServerAndLoad> list = convertToList(servers);
187 LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
188 List<RegionPlan> plans = loadBalancer.balanceCluster(servers);
189 List<ServerAndLoad> balancedCluster = reconcile(list, plans, servers);
190 LOG.info("Mock Balance : " + printMock(balancedCluster));
191 assertClusterAsBalanced(balancedCluster);
192 List<RegionPlan> secondPlans = loadBalancer.balanceCluster(servers);
193 assertNull(secondPlans);
194 for (Map.Entry<ServerName, List<HRegionInfo>> entry : servers.entrySet()) {
195 returnRegions(entry.getValue());
196 returnServer(entry.getKey());
197 }
198 }
199
200 }
201
202 @Test
203 public void testSkewCost() {
204 Configuration conf = HBaseConfiguration.create();
205 StochasticLoadBalancer.CostFunction
206 costFunction = new StochasticLoadBalancer.RegionCountSkewCostFunction(conf);
207 for (int[] mockCluster : clusterStateMocks) {
208 double cost = costFunction.cost(mockCluster(mockCluster));
209 assertTrue(cost >= 0);
210 assertTrue(cost <= 1.01);
211 }
212
213 assertEquals(0,
214 costFunction.cost(mockCluster(new int[]{0, 0, 0, 0, 1})), 0.01);
215 assertEquals(0,
216 costFunction.cost(mockCluster(new int[]{0, 0, 0, 1, 1})), 0.01);
217 assertEquals(0,
218 costFunction.cost(mockCluster(new int[]{0, 0, 1, 1, 1})), 0.01);
219 assertEquals(0,
220 costFunction.cost(mockCluster(new int[]{0, 1, 1, 1, 1})), 0.01);
221 assertEquals(0,
222 costFunction.cost(mockCluster(new int[]{1, 1, 1, 1, 1})), 0.01);
223 assertEquals(0,
224 costFunction.cost(mockCluster(new int[]{10, 10, 10, 10, 10})), 0.01);
225 assertEquals(1,
226 costFunction.cost(mockCluster(new int[]{10000, 0, 0, 0, 0})), 0.01);
227 }
228
229 @Test
230 public void testTableSkewCost() {
231 Configuration conf = HBaseConfiguration.create();
232 StochasticLoadBalancer.CostFunction
233 costFunction = new StochasticLoadBalancer.TableSkewCostFunction(conf);
234 for (int[] mockCluster : clusterStateMocks) {
235 BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster);
236 double cost = costFunction.cost(cluster);
237 assertTrue(cost >= 0);
238 assertTrue(cost <= 1.01);
239 }
240 }
241
242 @Test
243 public void testCostFromArray() {
244 Configuration conf = HBaseConfiguration.create();
245 StochasticLoadBalancer.CostFromRegionLoadFunction
246 costFunction = new StochasticLoadBalancer.MemstoreSizeCostFunction(conf);
247
248 double[] statOne = new double[100];
249 for (int i =0; i < 100; i++) {
250 statOne[i] = 10;
251 }
252 assertEquals(0, costFunction.costFromArray(statOne), 0.01);
253
254 double[] statTwo= new double[101];
255 for (int i =0; i < 100; i++) {
256 statTwo[i] = 0;
257 }
258 statTwo[100] = 101;
259 assertEquals(1, costFunction.costFromArray(statTwo), 0.01);
260
261 double[] statThree = new double[200];
262 for (int i =0; i < 100; i++) {
263 statThree[i] = (0);
264 statThree[i+100] = 100;
265 }
266 assertEquals(0.5, costFunction.costFromArray(statThree), 0.01);
267 }
268
269 @Test(timeout = 60000)
270 public void testLosingRs() throws Exception {
271 int numNodes = 3;
272 int numRegions = 20;
273 int numRegionsPerServer = 3;
274 int numTables = 2;
275
276 Map<ServerName, List<HRegionInfo>> serverMap =
277 createServerMap(numNodes, numRegions, numRegionsPerServer, numTables);
278 List<ServerAndLoad> list = convertToList(serverMap);
279
280
281 List<RegionPlan> plans = loadBalancer.balanceCluster(serverMap);
282 assertNotNull(plans);
283
284
285 List<ServerAndLoad> balancedCluster = reconcile(list, plans, serverMap);
286
287 assertClusterAsBalanced(balancedCluster);
288
289 ServerName sn = serverMap.keySet().toArray(new ServerName[serverMap.size()])[0];
290
291 ServerName deadSn = ServerName.valueOf(sn.getHostname(), sn.getPort(), sn.getStartcode() - 100);
292
293 serverMap.put(deadSn, new ArrayList<HRegionInfo>(0));
294
295 plans = loadBalancer.balanceCluster(serverMap);
296 assertNull(plans);
297 }
298
299 @Test (timeout = 60000)
300 public void testSmallCluster() {
301 int numNodes = 10;
302 int numRegions = 1000;
303 int numRegionsPerServer = 40;
304 int numTables = 10;
305 testWithCluster(numNodes, numRegions, numRegionsPerServer, numTables, true);
306 }
307
308 @Test (timeout = 60000)
309 public void testSmallCluster2() {
310 int numNodes = 20;
311 int numRegions = 2000;
312 int numRegionsPerServer = 40;
313 int numTables = 10;
314 testWithCluster(numNodes, numRegions, numRegionsPerServer, numTables, true);
315 }
316
317 @Test (timeout = 60000)
318 public void testSmallCluster3() {
319 int numNodes = 20;
320 int numRegions = 2000;
321 int numRegionsPerServer = 1;
322 int numTables = 10;
323 testWithCluster(numNodes, numRegions, numRegionsPerServer, numTables, false
324 }
325
326 @Test (timeout = 800000)
327 public void testMidCluster() {
328 int numNodes = 100;
329 int numRegions = 10000;
330 int numRegionsPerServer = 60;
331 int numTables = 40;
332 testWithCluster(numNodes, numRegions, numRegionsPerServer, numTables, true);
333 }
334
335 @Test (timeout = 800000)
336 public void testMidCluster2() {
337 int numNodes = 200;
338 int numRegions = 100000;
339 int numRegionsPerServer = 40;
340 int numTables = 400;
341 testWithCluster(numNodes,
342 numRegions,
343 numRegionsPerServer,
344 numTables,
345 false
346 }
347
348
349 @Test (timeout = 800000)
350 public void testMidCluster3() {
351 int numNodes = 100;
352 int numRegions = 2000;
353 int numRegionsPerServer = 9;
354 int numTables = 110;
355 testWithCluster(numNodes, numRegions, numRegionsPerServer, numTables, true);
356
357 }
358
359 @Test
360 public void testLargeCluster() {
361 int numNodes = 1000;
362 int numRegions = 100000;
363 int numRegionsPerServer = 80;
364 int numTables = 100;
365 testWithCluster(numNodes, numRegions, numRegionsPerServer, numTables, true);
366 }
367
368 protected void testWithCluster(int numNodes,
369 int numRegions,
370 int numRegionsPerServer,
371 int numTables,
372 boolean assertFullyBalanced) {
373 Map<ServerName, List<HRegionInfo>> serverMap =
374 createServerMap(numNodes, numRegions, numRegionsPerServer, numTables);
375
376 List<ServerAndLoad> list = convertToList(serverMap);
377 LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
378
379
380 List<RegionPlan> plans = loadBalancer.balanceCluster(serverMap);
381 assertNotNull(plans);
382
383
384 if (assertFullyBalanced) {
385
386 List<ServerAndLoad> balancedCluster = reconcile(list, plans, serverMap);
387
388
389 LOG.info("Mock Balance : " + printMock(balancedCluster));
390 assertClusterAsBalanced(balancedCluster);
391 List<RegionPlan> secondPlans = loadBalancer.balanceCluster(serverMap);
392 assertNull(secondPlans);
393 }
394 }
395
396 private Map<ServerName, List<HRegionInfo>> createServerMap(int numNodes,
397 int numRegions,
398 int numRegionsPerServer,
399 int numTables) {
400
401
402
403 int[] cluster = new int[numNodes];
404 for (int i =0; i < numNodes; i++) {
405 cluster[i] = numRegionsPerServer;
406 }
407 cluster[cluster.length - 1] = numRegions - ((cluster.length - 1) * numRegionsPerServer);
408 return mockClusterServers(cluster, numTables);
409 }
410 }