View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.balancer;
19  
20  import java.util.ArrayList;
21  import java.util.Arrays;
22  import java.util.List;
23  import java.util.Map;
24  import java.util.Queue;
25  import java.util.TreeMap;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.ClusterStatus;
31  import org.apache.hadoop.hbase.HBaseConfiguration;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.testclassification.MediumTests;
34  import org.apache.hadoop.hbase.RegionLoad;
35  import org.apache.hadoop.hbase.ServerLoad;
36  import org.apache.hadoop.hbase.ServerName;
37  import org.apache.hadoop.hbase.master.RegionPlan;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.junit.BeforeClass;
40  import org.junit.Test;
41  import org.junit.experimental.categories.Category;
42  
43  import static org.junit.Assert.assertEquals;
44  import static org.junit.Assert.assertNotNull;
45  import static org.junit.Assert.assertNull;
46  import static org.junit.Assert.assertTrue;
47  import static org.mockito.Mockito.mock;
48  import static org.mockito.Mockito.when;
49  
50  @Category(MediumTests.class)
51  public class TestStochasticLoadBalancer extends BalancerTestBase {
52    public static final String REGION_KEY = "testRegion";
53    private static StochasticLoadBalancer loadBalancer;
54    private static final Log LOG = LogFactory.getLog(TestStochasticLoadBalancer.class);
55  
56    @BeforeClass
57    public static void beforeAllTests() throws Exception {
58      Configuration conf = HBaseConfiguration.create();
59      conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f);
60      conf.setFloat("hbase.regions.slop", 0.0f);
61      loadBalancer = new StochasticLoadBalancer();
62      loadBalancer.setConf(conf);
63    }
64  
65    int[] largeCluster = new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
66        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
67        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
68        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
69        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
70        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
71        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
72        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
73        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
74        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
75        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
76        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
77        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
78        0, 0, 0, 0, 0, 56 };
79  
80    // int[testnum][servernumber] -> numregions
81    int[][] clusterStateMocks = new int[][]{
82        // 1 node
83        new int[]{0},
84        new int[]{1},
85        new int[]{10},
86        // 2 node
87        new int[]{0, 0},
88        new int[]{2, 0},
89        new int[]{2, 1},
90        new int[]{2, 2},
91        new int[]{2, 3},
92        new int[]{2, 4},
93        new int[]{1, 1},
94        new int[]{0, 1},
95        new int[]{10, 1},
96        new int[]{514, 1432},
97        new int[]{48, 53},
98        // 3 node
99        new int[]{0, 1, 2},
100       new int[]{1, 2, 3},
101       new int[]{0, 2, 2},
102       new int[]{0, 3, 0},
103       new int[]{0, 4, 0},
104       new int[]{20, 20, 0},
105       // 4 node
106       new int[]{0, 1, 2, 3},
107       new int[]{4, 0, 0, 0},
108       new int[]{5, 0, 0, 0},
109       new int[]{6, 6, 0, 0},
110       new int[]{6, 2, 0, 0},
111       new int[]{6, 1, 0, 0},
112       new int[]{6, 0, 0, 0},
113       new int[]{4, 4, 4, 7},
114       new int[]{4, 4, 4, 8},
115       new int[]{0, 0, 0, 7},
116       // 5 node
117       new int[]{1, 1, 1, 1, 4},
118       // more nodes
119       new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
120       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 10},
121       new int[]{6, 6, 5, 6, 6, 6, 6, 6, 6, 1},
122       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 54},
123       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 55},
124       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 56},
125       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 16},
126       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 8},
127       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 9},
128       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 10},
129       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 123},
130       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 155},
131       new int[]{10, 7, 12, 8, 11, 10, 9, 14},
132       new int[]{13, 14, 6, 10, 10, 10, 8, 10},
133       new int[]{130, 14, 60, 10, 100, 10, 80, 10},
134       new int[]{130, 140, 60, 100, 100, 100, 80, 100},
135       largeCluster,
136 
137   };
138 
139   @Test
140   public void testKeepRegionLoad() throws Exception {
141 
142     ServerName sn = ServerName.valueOf("test:8080", 100);
143     int numClusterStatusToAdd = 20000;
144     for (int i = 0; i < numClusterStatusToAdd; i++) {
145       ServerLoad sl = mock(ServerLoad.class);
146 
147       RegionLoad rl = mock(RegionLoad.class);
148       when(rl.getStores()).thenReturn(i);
149 
150       Map<byte[], RegionLoad> regionLoadMap =
151           new TreeMap<byte[], RegionLoad>(Bytes.BYTES_COMPARATOR);
152       regionLoadMap.put(Bytes.toBytes(REGION_KEY), rl);
153       when(sl.getRegionsLoad()).thenReturn(regionLoadMap);
154 
155       ClusterStatus clusterStatus = mock(ClusterStatus.class);
156       when(clusterStatus.getServers()).thenReturn(Arrays.asList(sn));
157       when(clusterStatus.getLoad(sn)).thenReturn(sl);
158 
159       loadBalancer.setClusterStatus(clusterStatus);
160     }
161     assertTrue(loadBalancer.loads.get(REGION_KEY) != null);
162     assertTrue(loadBalancer.loads.get(REGION_KEY).size() == 15);
163 
164     Queue<RegionLoad> loads = loadBalancer.loads.get(REGION_KEY);
165     int i = 0;
166     while(loads.size() > 0) {
167       RegionLoad rl = loads.remove();
168       assertEquals(i + (numClusterStatusToAdd - 15), rl.getStores());
169       i ++;
170     }
171   }
172 
173   /**
174    * Test the load balancing algorithm.
175    *
176    * Invariant is that all servers should be hosting either floor(average) or
177    * ceiling(average)
178    *
179    * @throws Exception
180    */
181   @Test
182   public void testBalanceCluster() throws Exception {
183 
184     for (int[] mockCluster : clusterStateMocks) {
185       Map<ServerName, List<HRegionInfo>> servers = mockClusterServers(mockCluster);
186       List<ServerAndLoad> list = convertToList(servers);
187       LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
188       List<RegionPlan> plans = loadBalancer.balanceCluster(servers);
189       List<ServerAndLoad> balancedCluster = reconcile(list, plans, servers);
190       LOG.info("Mock Balance : " + printMock(balancedCluster));
191       assertClusterAsBalanced(balancedCluster);
192       List<RegionPlan> secondPlans =  loadBalancer.balanceCluster(servers);
193       assertNull(secondPlans);
194       for (Map.Entry<ServerName, List<HRegionInfo>> entry : servers.entrySet()) {
195         returnRegions(entry.getValue());
196         returnServer(entry.getKey());
197       }
198     }
199 
200   }
201 
202   @Test
203   public void testSkewCost() {
204     Configuration conf = HBaseConfiguration.create();
205     StochasticLoadBalancer.CostFunction
206         costFunction = new StochasticLoadBalancer.RegionCountSkewCostFunction(conf);
207     for (int[] mockCluster : clusterStateMocks) {
208       double cost = costFunction.cost(mockCluster(mockCluster));
209       assertTrue(cost >= 0);
210       assertTrue(cost <= 1.01);
211     }
212 
213     assertEquals(0,
214         costFunction.cost(mockCluster(new int[]{0, 0, 0, 0, 1})), 0.01);
215     assertEquals(0,
216         costFunction.cost(mockCluster(new int[]{0, 0, 0, 1, 1})), 0.01);
217     assertEquals(0,
218         costFunction.cost(mockCluster(new int[]{0, 0, 1, 1, 1})), 0.01);
219     assertEquals(0,
220         costFunction.cost(mockCluster(new int[]{0, 1, 1, 1, 1})), 0.01);
221     assertEquals(0,
222         costFunction.cost(mockCluster(new int[]{1, 1, 1, 1, 1})), 0.01);
223     assertEquals(0,
224         costFunction.cost(mockCluster(new int[]{10, 10, 10, 10, 10})), 0.01);
225     assertEquals(1,
226         costFunction.cost(mockCluster(new int[]{10000, 0, 0, 0, 0})), 0.01);
227   }
228 
229   @Test
230   public void testTableSkewCost() {
231     Configuration conf = HBaseConfiguration.create();
232     StochasticLoadBalancer.CostFunction
233         costFunction = new StochasticLoadBalancer.TableSkewCostFunction(conf);
234     for (int[] mockCluster : clusterStateMocks) {
235       BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster);
236       double cost = costFunction.cost(cluster);
237       assertTrue(cost >= 0);
238       assertTrue(cost <= 1.01);
239     }
240   }
241 
242   @Test
243   public void testCostFromArray() {
244     Configuration conf = HBaseConfiguration.create();
245     StochasticLoadBalancer.CostFromRegionLoadFunction
246         costFunction = new StochasticLoadBalancer.MemstoreSizeCostFunction(conf);
247 
248     double[] statOne = new double[100];
249     for (int i =0; i < 100; i++) {
250       statOne[i] = 10;
251     }
252     assertEquals(0, costFunction.costFromArray(statOne), 0.01);
253 
254     double[] statTwo= new double[101];
255     for (int i =0; i < 100; i++) {
256       statTwo[i] = 0;
257     }
258     statTwo[100] = 101;
259     assertEquals(1, costFunction.costFromArray(statTwo), 0.01);
260 
261     double[] statThree = new double[200];
262     for (int i =0; i < 100; i++) {
263       statThree[i] = (0);
264       statThree[i+100] = 100;
265     }
266     assertEquals(0.5, costFunction.costFromArray(statThree), 0.01);
267   }
268 
269   @Test(timeout =  60000)
270   public void testLosingRs() throws Exception {
271     int numNodes = 3;
272     int numRegions = 20;
273     int numRegionsPerServer = 3; //all servers except one
274     int numTables = 2;
275 
276     Map<ServerName, List<HRegionInfo>> serverMap =
277         createServerMap(numNodes, numRegions, numRegionsPerServer, numTables);
278     List<ServerAndLoad> list = convertToList(serverMap);
279 
280 
281     List<RegionPlan> plans = loadBalancer.balanceCluster(serverMap);
282     assertNotNull(plans);
283 
284     // Apply the plan to the mock cluster.
285     List<ServerAndLoad> balancedCluster = reconcile(list, plans, serverMap);
286 
287     assertClusterAsBalanced(balancedCluster);
288 
289     ServerName sn = serverMap.keySet().toArray(new ServerName[serverMap.size()])[0];
290 
291     ServerName deadSn = ServerName.valueOf(sn.getHostname(), sn.getPort(), sn.getStartcode() - 100);
292 
293     serverMap.put(deadSn, new ArrayList<HRegionInfo>(0));
294 
295     plans = loadBalancer.balanceCluster(serverMap);
296     assertNull(plans);
297   }
298 
299   @Test (timeout = 60000)
300   public void testSmallCluster() {
301     int numNodes = 10;
302     int numRegions = 1000;
303     int numRegionsPerServer = 40; //all servers except one
304     int numTables = 10;
305     testWithCluster(numNodes, numRegions, numRegionsPerServer, numTables, true);
306   }
307 
308   @Test (timeout = 60000)
309   public void testSmallCluster2() {
310     int numNodes = 20;
311     int numRegions = 2000;
312     int numRegionsPerServer = 40; //all servers except one
313     int numTables = 10;
314     testWithCluster(numNodes, numRegions, numRegionsPerServer, numTables, true);
315   }
316 
317   @Test (timeout = 60000)
318   public void testSmallCluster3() {
319     int numNodes = 20;
320     int numRegions = 2000;
321     int numRegionsPerServer = 1; // all servers except one
322     int numTables = 10;
323     testWithCluster(numNodes, numRegions, numRegionsPerServer, numTables, false /* max moves */);
324   }
325 
326   @Test (timeout = 800000)
327   public void testMidCluster() {
328     int numNodes = 100;
329     int numRegions = 10000;
330     int numRegionsPerServer = 60; // all servers except one
331     int numTables = 40;
332     testWithCluster(numNodes, numRegions, numRegionsPerServer, numTables, true);
333   }
334 
335   @Test (timeout = 800000)
336   public void testMidCluster2() {
337     int numNodes = 200;
338     int numRegions = 100000;
339     int numRegionsPerServer = 40; // all servers except one
340     int numTables = 400;
341     testWithCluster(numNodes,
342         numRegions,
343         numRegionsPerServer,
344         numTables,
345         false /* num large num regions means may not always get to best balance with one run */);
346   }
347 
348 
349   @Test (timeout = 800000)
350   public void testMidCluster3() {
351     int numNodes = 100;
352     int numRegions = 2000;
353     int numRegionsPerServer = 9; // all servers except one
354     int numTables = 110;
355     testWithCluster(numNodes, numRegions, numRegionsPerServer, numTables, true);
356     // TODO(eclark): Make sure that the tables are well distributed.
357   }
358 
359   @Test
360   public void testLargeCluster() {
361     int numNodes = 1000;
362     int numRegions = 100000; //100 regions per RS
363     int numRegionsPerServer = 80; //all servers except one
364     int numTables = 100;
365     testWithCluster(numNodes, numRegions, numRegionsPerServer, numTables, true);
366   }
367 
368   protected void testWithCluster(int numNodes,
369                                  int numRegions,
370                                  int numRegionsPerServer,
371                                  int numTables,
372                                  boolean assertFullyBalanced) {
373     Map<ServerName, List<HRegionInfo>> serverMap =
374         createServerMap(numNodes, numRegions, numRegionsPerServer, numTables);
375 
376     List<ServerAndLoad> list = convertToList(serverMap);
377     LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
378 
379     // Run the balancer.
380     List<RegionPlan> plans = loadBalancer.balanceCluster(serverMap);
381     assertNotNull(plans);
382 
383     // Check to see that this actually got to a stable place.
384     if (assertFullyBalanced) {
385       // Apply the plan to the mock cluster.
386       List<ServerAndLoad> balancedCluster = reconcile(list, plans, serverMap);
387 
388       // Print out the cluster loads to make debugging easier.
389       LOG.info("Mock Balance : " + printMock(balancedCluster));
390       assertClusterAsBalanced(balancedCluster);
391       List<RegionPlan> secondPlans =  loadBalancer.balanceCluster(serverMap);
392       assertNull(secondPlans);
393     }
394   }
395 
396   private Map<ServerName, List<HRegionInfo>> createServerMap(int numNodes,
397                                                              int numRegions,
398                                                              int numRegionsPerServer,
399                                                              int numTables) {
400     //construct a cluster of numNodes, having  a total of numRegions. Each RS will hold
401     //numRegionsPerServer many regions except for the last one, which will host all the
402     //remaining regions
403     int[] cluster = new int[numNodes];
404     for (int i =0; i < numNodes; i++) {
405       cluster[i] = numRegionsPerServer;
406     }
407     cluster[cluster.length - 1] = numRegions - ((cluster.length - 1) * numRegionsPerServer);
408     return mockClusterServers(cluster, numTables);
409   }
410 }