1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase;
19
20
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23 import org.apache.hadoop.conf.Configuration;
24 import org.apache.hadoop.hbase.catalog.CatalogTracker;
25 import org.apache.hadoop.hbase.executor.EventType;
26 import org.apache.hadoop.hbase.executor.ExecutorService;
27 import org.apache.hadoop.hbase.executor.ExecutorType;
28 import org.apache.hadoop.hbase.master.AssignmentManager;
29 import org.apache.hadoop.hbase.master.HMaster;
30 import org.apache.hadoop.hbase.master.LoadBalancer;
31 import org.apache.hadoop.hbase.master.RegionPlan;
32 import org.apache.hadoop.hbase.master.RegionState;
33 import org.apache.hadoop.hbase.master.ServerManager;
34 import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
35 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
36 import org.apache.hadoop.hbase.testclassification.MediumTests;
37 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
38 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
39 import org.junit.AfterClass;
40 import org.junit.BeforeClass;
41 import org.junit.Test;
42 import org.junit.experimental.categories.Category;
43 import org.mockito.Mockito;
44
45 import java.util.ArrayList;
46 import java.util.HashMap;
47 import java.util.HashSet;
48 import java.util.List;
49 import java.util.Map;
50 import java.util.Map.Entry;
51 import java.util.Set;
52
53 import static org.junit.Assert.assertFalse;
54 import static org.junit.Assert.assertNotEquals;
55 import static org.junit.Assert.assertTrue;
56
57
58
59
60
61 @Category(MediumTests.class)
62 public class TestDrainingServer {
63 private static final Log LOG = LogFactory.getLog(TestDrainingServer.class);
64 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
65 private Abortable abortable = new Abortable() {
66 @Override
67 public boolean isAborted() {
68 return false;
69 }
70
71 @Override
72 public void abort(String why, Throwable e) {
73 }
74 };
75
76 @AfterClass
77 public static void afterClass() throws Exception {
78 TEST_UTIL.shutdownMiniZKCluster();
79 }
80
81 @BeforeClass
82 public static void beforeClass() throws Exception {
83 TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
84 TEST_UTIL.startMiniZKCluster();
85 }
86
87 @Test
88 public void testAssignmentManagerDoesntUseDrainingServer() throws Exception {
89 AssignmentManager am;
90 Configuration conf = TEST_UTIL.getConfiguration();
91 final HMaster master = Mockito.mock(HMaster.class);
92 final Server server = Mockito.mock(Server.class);
93 final ServerManager serverManager = Mockito.mock(ServerManager.class);
94 final ServerName SERVERNAME_A = ServerName.valueOf("mockserver_a.org", 1000, 8000);
95 final ServerName SERVERNAME_B = ServerName.valueOf("mockserver_b.org", 1001, 8000);
96 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
97 CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class);
98 final HRegionInfo REGIONINFO = new HRegionInfo(TableName.valueOf("table_test"),
99 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
100
101 ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
102 "zkWatcher-Test", abortable, true);
103
104 Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
105
106 onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
107 onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
108
109 Mockito.when(server.getConfiguration()).thenReturn(conf);
110 Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
111 Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);
112
113 Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
114 Mockito.when(serverManager.getOnlineServersList())
115 .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
116
117 Mockito.when(serverManager.createDestinationServersList())
118 .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
119 Mockito.when(serverManager.createDestinationServersList(null))
120 .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
121
122 for (ServerName sn : onlineServers.keySet()) {
123 Mockito.when(serverManager.isServerOnline(sn)).thenReturn(true);
124 Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1)).thenReturn(true);
125 Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1, null, false)).thenReturn(true);
126 Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, new ArrayList<ServerName>()))
127 .thenReturn(RegionOpeningState.OPENED);
128 Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, null))
129 .thenReturn(RegionOpeningState.OPENED);
130 Mockito.when(serverManager.addServerToDrainList(sn)).thenReturn(true);
131 }
132
133 Mockito.when(master.getServerManager()).thenReturn(serverManager);
134
135 am = new AssignmentManager(server, serverManager, catalogTracker,
136 balancer, startupMasterExecutor("mockExecutorService"), null, null);
137
138 Mockito.when(master.getAssignmentManager()).thenReturn(am);
139 Mockito.when(master.getZooKeeperWatcher()).thenReturn(zkWatcher);
140 Mockito.when(master.getZooKeeper()).thenReturn(zkWatcher);
141
142 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));
143
144 zkWatcher.registerListenerFirst(am);
145
146 addServerToDrainedList(SERVERNAME_A, onlineServers, serverManager);
147
148 am.assign(REGIONINFO, true);
149
150 setRegionOpenedOnZK(zkWatcher, SERVERNAME_A, REGIONINFO);
151 setRegionOpenedOnZK(zkWatcher, SERVERNAME_B, REGIONINFO);
152
153 am.waitForAssignment(REGIONINFO);
154
155 assertTrue(am.getRegionStates().isRegionOnline(REGIONINFO));
156 assertNotEquals(am.getRegionStates().getRegionServerOfRegion(REGIONINFO), SERVERNAME_A);
157 }
158
159 @Test
160 public void testAssignmentManagerDoesntUseDrainedServerWithBulkAssign() throws Exception {
161 Configuration conf = TEST_UTIL.getConfiguration();
162 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
163 CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class);
164 AssignmentManager am;
165 final HMaster master = Mockito.mock(HMaster.class);
166 final Server server = Mockito.mock(Server.class);
167 final ServerManager serverManager = Mockito.mock(ServerManager.class);
168 final ServerName SERVERNAME_A = ServerName.valueOf("mockserverbulk_a.org", 1000, 8000);
169 final ServerName SERVERNAME_B = ServerName.valueOf("mockserverbulk_b.org", 1001, 8000);
170 final ServerName SERVERNAME_C = ServerName.valueOf("mockserverbulk_c.org", 1002, 8000);
171 final ServerName SERVERNAME_D = ServerName.valueOf("mockserverbulk_d.org", 1003, 8000);
172 final ServerName SERVERNAME_E = ServerName.valueOf("mockserverbulk_e.org", 1004, 8000);
173 final Map<HRegionInfo, ServerName> bulk = new HashMap<HRegionInfo, ServerName>();
174
175 Set<ServerName> bunchServersAssigned = new HashSet<ServerName>();
176
177 HRegionInfo REGIONINFO_A = new HRegionInfo(TableName.valueOf("table_A"),
178 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
179 HRegionInfo REGIONINFO_B = new HRegionInfo(TableName.valueOf("table_B"),
180 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
181 HRegionInfo REGIONINFO_C = new HRegionInfo(TableName.valueOf("table_C"),
182 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
183 HRegionInfo REGIONINFO_D = new HRegionInfo(TableName.valueOf("table_D"),
184 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
185 HRegionInfo REGIONINFO_E = new HRegionInfo(TableName.valueOf("table_E"),
186 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
187
188 Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
189 List<ServerName> drainedServers = new ArrayList<ServerName>();
190
191 onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
192 onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
193 onlineServers.put(SERVERNAME_C, ServerLoad.EMPTY_SERVERLOAD);
194 onlineServers.put(SERVERNAME_D, ServerLoad.EMPTY_SERVERLOAD);
195 onlineServers.put(SERVERNAME_E, ServerLoad.EMPTY_SERVERLOAD);
196
197 bulk.put(REGIONINFO_A, SERVERNAME_A);
198 bulk.put(REGIONINFO_B, SERVERNAME_B);
199 bulk.put(REGIONINFO_C, SERVERNAME_C);
200 bulk.put(REGIONINFO_D, SERVERNAME_D);
201 bulk.put(REGIONINFO_E, SERVERNAME_E);
202
203 ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
204 "zkWatcher-BulkAssignTest", abortable, true);
205
206 Mockito.when(server.getConfiguration()).thenReturn(conf);
207 Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
208 Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);
209
210 Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
211 Mockito.when(serverManager.getOnlineServersList()).thenReturn(
212 new ArrayList<ServerName>(onlineServers.keySet()));
213
214 Mockito.when(serverManager.createDestinationServersList()).thenReturn(
215 new ArrayList<ServerName>(onlineServers.keySet()));
216 Mockito.when(serverManager.createDestinationServersList(null)).thenReturn(
217 new ArrayList<ServerName>(onlineServers.keySet()));
218
219 for (Entry<HRegionInfo, ServerName> entry : bulk.entrySet()) {
220 Mockito.when(serverManager.isServerOnline(entry.getValue())).thenReturn(true);
221 Mockito.when(serverManager.sendRegionClose(entry.getValue(),
222 entry.getKey(), -1)).thenReturn(true);
223 Mockito.when(serverManager.sendRegionOpen(entry.getValue(),
224 entry.getKey(), -1, null)).thenReturn(RegionOpeningState.OPENED);
225 Mockito.when(serverManager.addServerToDrainList(entry.getValue())).thenReturn(true);
226 }
227
228 Mockito.when(master.getServerManager()).thenReturn(serverManager);
229
230 drainedServers.add(SERVERNAME_A);
231 drainedServers.add(SERVERNAME_B);
232 drainedServers.add(SERVERNAME_C);
233 drainedServers.add(SERVERNAME_D);
234
235 am = new AssignmentManager(server, serverManager, catalogTracker,
236 balancer, startupMasterExecutor("mockExecutorServiceBulk"), null, null);
237
238 Mockito.when(master.getAssignmentManager()).thenReturn(am);
239
240 zkWatcher.registerListener(am);
241
242 for (ServerName drained : drainedServers) {
243 addServerToDrainedList(drained, onlineServers, serverManager);
244 }
245
246 am.assign(bulk);
247
248 Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
249 for (Entry<String, RegionState> entry : regionsInTransition.entrySet()) {
250 setRegionOpenedOnZK(zkWatcher, entry.getValue().getServerName(),
251 entry.getValue().getRegion());
252 }
253
254 am.waitForAssignment(REGIONINFO_A);
255 am.waitForAssignment(REGIONINFO_B);
256 am.waitForAssignment(REGIONINFO_C);
257 am.waitForAssignment(REGIONINFO_D);
258 am.waitForAssignment(REGIONINFO_E);
259
260 Map<HRegionInfo, ServerName> regionAssignments = am.getRegionStates().getRegionAssignments();
261 for (Entry<HRegionInfo, ServerName> entry : regionAssignments.entrySet()) {
262 LOG.info("Region Assignment: "
263 + entry.getKey().getRegionNameAsString() + " Server: " + entry.getValue());
264 bunchServersAssigned.add(entry.getValue());
265 }
266
267 for (ServerName sn : drainedServers) {
268 assertFalse(bunchServersAssigned.contains(sn));
269 }
270 }
271
272 private void addServerToDrainedList(ServerName serverName,
273 Map<ServerName, ServerLoad> onlineServers, ServerManager serverManager) {
274 onlineServers.remove(serverName);
275 List<ServerName> availableServers = new ArrayList<ServerName>(onlineServers.keySet());
276 Mockito.when(serverManager.createDestinationServersList()).thenReturn(availableServers);
277 Mockito.when(serverManager.createDestinationServersList(null)).thenReturn(availableServers);
278 }
279
280 private void setRegionOpenedOnZK(final ZooKeeperWatcher zkWatcher, final ServerName serverName,
281 HRegionInfo hregionInfo) throws Exception {
282 int version = ZKAssign.getVersion(zkWatcher, hregionInfo);
283 int versionTransition = ZKAssign.transitionNode(zkWatcher,
284 hregionInfo, serverName, EventType.M_ZK_REGION_OFFLINE,
285 EventType.RS_ZK_REGION_OPENING, version);
286 ZKAssign.transitionNodeOpened(zkWatcher, hregionInfo, serverName, versionTransition);
287 }
288
289 private ExecutorService startupMasterExecutor(final String name) {
290 ExecutorService executor = new ExecutorService(name);
291 executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
292 executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
293 executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
294 executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
295 return executor;
296 }
297 }