View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.balancer;
19  
20  import static org.junit.Assert.assertTrue;
21  
22  import java.util.ArrayList;
23  import java.util.HashMap;
24  import java.util.LinkedList;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.Queue;
28  import java.util.Random;
29  import java.util.SortedSet;
30  import java.util.TreeMap;
31  import java.util.TreeSet;
32  
33  import org.apache.hadoop.hbase.TableName;
34  import org.apache.hadoop.hbase.HRegionInfo;
35  import org.apache.hadoop.hbase.ServerName;
36  import org.apache.hadoop.hbase.master.RegionPlan;
37  import org.apache.hadoop.hbase.util.Bytes;
38  
39  /**
40   * Class used to be the base of unit tests on load balancers. It gives helper
41   * methods to create maps of {@link ServerName} to lists of {@link HRegionInfo}
42   * and to check list of region plans.
43   *
44   */
45  public class BalancerTestBase {
46  
47    protected static Random rand = new Random();
48    static int regionId = 0;
49  
50    /**
51     * Invariant is that all servers have between floor(avg) and ceiling(avg)
52     * number of regions.
53     */
54    public void assertClusterAsBalanced(List<ServerAndLoad> servers) {
55      int numServers = servers.size();
56      int numRegions = 0;
57      int maxRegions = 0;
58      int minRegions = Integer.MAX_VALUE;
59      for (ServerAndLoad server : servers) {
60        int nr = server.getLoad();
61        if (nr > maxRegions) {
62          maxRegions = nr;
63        }
64        if (nr < minRegions) {
65          minRegions = nr;
66        }
67        numRegions += nr;
68      }
69      if (maxRegions - minRegions < 2) {
70        // less than 2 between max and min, can't balance
71        return;
72      }
73      int min = numRegions / numServers;
74      int max = numRegions % numServers == 0 ? min : min + 1;
75  
76      for (ServerAndLoad server : servers) {
77        assertTrue(server.getLoad() >= 0);
78        assertTrue(server.getLoad() <= max);
79        assertTrue(server.getLoad() >= min);
80      }
81    }
82  
83    protected String printStats(List<ServerAndLoad> servers) {
84      int numServers = servers.size();
85      int totalRegions = 0;
86      for (ServerAndLoad server : servers) {
87        totalRegions += server.getLoad();
88      }
89      float average = (float) totalRegions / numServers;
90      int max = (int) Math.ceil(average);
91      int min = (int) Math.floor(average);
92      return "[srvr=" + numServers + " rgns=" + totalRegions + " avg=" + average + " max=" + max
93          + " min=" + min + "]";
94    }
95  
96    protected List<ServerAndLoad> convertToList(final Map<ServerName, List<HRegionInfo>> servers) {
97      List<ServerAndLoad> list = new ArrayList<ServerAndLoad>(servers.size());
98      for (Map.Entry<ServerName, List<HRegionInfo>> e : servers.entrySet()) {
99        list.add(new ServerAndLoad(e.getKey(), e.getValue().size()));
100     }
101     return list;
102   }
103 
104   protected String printMock(List<ServerAndLoad> balancedCluster) {
105     SortedSet<ServerAndLoad> sorted = new TreeSet<ServerAndLoad>(balancedCluster);
106     ServerAndLoad[] arr = sorted.toArray(new ServerAndLoad[sorted.size()]);
107     StringBuilder sb = new StringBuilder(sorted.size() * 4 + 4);
108     sb.append("{ ");
109     for (int i = 0; i < arr.length; i++) {
110       if (i != 0) {
111         sb.append(" , ");
112       }
113       sb.append(arr[i].getServerName().getHostname());
114       sb.append(":");
115       sb.append(arr[i].getLoad());
116     }
117     sb.append(" }");
118     return sb.toString();
119   }
120 
121   /**
122    * This assumes the RegionPlan HSI instances are the same ones in the map, so
123    * actually no need to even pass in the map, but I think it's clearer.
124    *
125    * @param list
126    * @param plans
127    * @return
128    */
129   protected List<ServerAndLoad> reconcile(List<ServerAndLoad> list,
130                                           List<RegionPlan> plans,
131                                           Map<ServerName, List<HRegionInfo>> servers) {
132     List<ServerAndLoad> result = new ArrayList<ServerAndLoad>(list.size());
133 
134     Map<ServerName, ServerAndLoad> map = new HashMap<ServerName, ServerAndLoad>(list.size());
135     for (ServerAndLoad sl : list) {
136       map.put(sl.getServerName(), sl);
137     }
138     if (plans != null) {
139       for (RegionPlan plan : plans) {
140         ServerName source = plan.getSource();
141 
142         updateLoad(map, source, -1);
143         ServerName destination = plan.getDestination();
144         updateLoad(map, destination, +1);
145 
146         servers.get(source).remove(plan.getRegionInfo());
147         servers.get(destination).add(plan.getRegionInfo());
148       }
149     }
150     result.clear();
151     result.addAll(map.values());
152     return result;
153   }
154 
155   protected void updateLoad(final Map<ServerName, ServerAndLoad> map,
156                             final ServerName sn,
157                             final int diff) {
158     ServerAndLoad sal = map.get(sn);
159     if (sal == null) sal = new ServerAndLoad(sn, 0);
160     sal = new ServerAndLoad(sn, sal.getLoad() + diff);
161     map.put(sn, sal);
162   }
163 
164   protected Map<ServerName, List<HRegionInfo>> mockClusterServers(int[] mockCluster) {
165     return mockClusterServers(mockCluster, -1);
166   }
167 
168   protected BaseLoadBalancer.Cluster mockCluster(int[] mockCluster) {
169     return new BaseLoadBalancer.Cluster(mockClusterServers(mockCluster, -1), null, null);
170   }
171 
172   protected Map<ServerName, List<HRegionInfo>> mockClusterServers(int[] mockCluster, int numTables) {
173     int numServers = mockCluster.length;
174     Map<ServerName, List<HRegionInfo>> servers = new TreeMap<ServerName, List<HRegionInfo>>();
175     for (int i = 0; i < numServers; i++) {
176       int numRegions = mockCluster[i];
177       ServerAndLoad sal = randomServer(0);
178       List<HRegionInfo> regions = randomRegions(numRegions, numTables);
179       servers.put(sal.getServerName(), regions);
180     }
181     return servers;
182   }
183 
184   private Queue<HRegionInfo> regionQueue = new LinkedList<HRegionInfo>();
185 
186   protected List<HRegionInfo> randomRegions(int numRegions) {
187     return randomRegions(numRegions, -1);
188   }
189 
190   protected List<HRegionInfo> randomRegions(int numRegions, int numTables) {
191     List<HRegionInfo> regions = new ArrayList<HRegionInfo>(numRegions);
192     byte[] start = new byte[16];
193     byte[] end = new byte[16];
194     rand.nextBytes(start);
195     rand.nextBytes(end);
196     for (int i = 0; i < numRegions; i++) {
197       if (!regionQueue.isEmpty()) {
198         regions.add(regionQueue.poll());
199         continue;
200       }
201       Bytes.putInt(start, 0, numRegions << 1);
202       Bytes.putInt(end, 0, (numRegions << 1) + 1);
203       TableName tableName =
204           TableName.valueOf("table" + (numTables > 0 ? rand.nextInt(numTables) : i));
205       HRegionInfo hri = new HRegionInfo(tableName, start, end, false, regionId++);
206       regions.add(hri);
207     }
208     return regions;
209   }
210 
211   protected void returnRegions(List<HRegionInfo> regions) {
212     regionQueue.addAll(regions);
213   }
214 
215   private Queue<ServerName> serverQueue = new LinkedList<ServerName>();
216 
217   protected ServerAndLoad randomServer(final int numRegionsPerServer) {
218     if (!this.serverQueue.isEmpty()) {
219       ServerName sn = this.serverQueue.poll();
220       return new ServerAndLoad(sn, numRegionsPerServer);
221     }
222     String host = "srv" + rand.nextInt(100000);
223     int port = rand.nextInt(60000);
224     long startCode = rand.nextLong();
225     ServerName sn = ServerName.valueOf(host, port, startCode);
226     return new ServerAndLoad(sn, numRegionsPerServer);
227   }
228 
229   protected List<ServerAndLoad> randomServers(int numServers, int numRegionsPerServer) {
230     List<ServerAndLoad> servers = new ArrayList<ServerAndLoad>(numServers);
231     for (int i = 0; i < numServers; i++) {
232       servers.add(randomServer(numRegionsPerServer));
233     }
234     return servers;
235   }
236 
237   protected void returnServer(ServerName server) {
238     serverQueue.add(server);
239   }
240 
241   protected void returnServers(List<ServerName> servers) {
242     this.serverQueue.addAll(servers);
243   }
244 
245 }