View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.IOException;
22  import java.net.InetAddress;
23  import java.util.ArrayList;
24  import java.util.Collections;
25  import java.util.HashMap;
26  import java.util.HashSet;
27  import java.util.Iterator;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Map.Entry;
31  import java.util.Set;
32  import java.util.SortedMap;
33  import java.util.concurrent.ConcurrentHashMap;
34  import java.util.concurrent.ConcurrentSkipListMap;
35  import java.util.concurrent.CopyOnWriteArrayList;
36  
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.hbase.classification.InterfaceAudience;
40  import org.apache.hadoop.conf.Configuration;
41  import org.apache.hadoop.hbase.ClockOutOfSyncException;
42  import org.apache.hadoop.hbase.HRegionInfo;
43  import org.apache.hadoop.hbase.RegionLoad;
44  import org.apache.hadoop.hbase.Server;
45  import org.apache.hadoop.hbase.ServerLoad;
46  import org.apache.hadoop.hbase.ServerName;
47  import org.apache.hadoop.hbase.YouAreDeadException;
48  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
49  import org.apache.hadoop.hbase.client.HConnection;
50  import org.apache.hadoop.hbase.client.HConnectionManager;
51  import org.apache.hadoop.hbase.client.RetriesExhaustedException;
52  import org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler;
53  import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
54  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
55  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
56  import org.apache.hadoop.hbase.protobuf.RequestConverter;
57  import org.apache.hadoop.hbase.protobuf.ResponseConverter;
58  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
59  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
60  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
61  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
62  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
63  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
64  import org.apache.hadoop.hbase.util.Bytes;
65  import org.apache.hadoop.hbase.util.Triple;
66  import org.apache.hadoop.hbase.util.RetryCounter;
67  import org.apache.hadoop.hbase.util.RetryCounterFactory;
68  
69  import com.google.common.annotations.VisibleForTesting;
70  import com.google.protobuf.ServiceException;
71  
72  /**
73   * The ServerManager class manages info about region servers.
74   * <p>
75   * Maintains lists of online and dead servers.  Processes the startups,
76   * shutdowns, and deaths of region servers.
77   * <p>
78   * Servers are distinguished in two different ways.  A given server has a
79   * location, specified by hostname and port, and of which there can only be one
80   * online at any given time.  A server instance is specified by the location
81   * (hostname and port) as well as the startcode (timestamp from when the server
82   * was started).  This is used to differentiate a restarted instance of a given
83   * server from the original instance.
84   * <p>
85   * If a sever is known not to be running any more, it is called dead. The dead
86   * server needs to be handled by a ServerShutdownHandler.  If the handler is not
87   * enabled yet, the server can't be handled right away so it is queued up.
88   * After the handler is enabled, the server will be submitted to a handler to handle.
89   * However, the handler may be just partially enabled.  If so,
90   * the server cannot be fully processed, and be queued up for further processing.
91   * A server is fully processed only after the handler is fully enabled
92   * and has completed the handling.
93   */
94  @InterfaceAudience.Private
95  public class ServerManager {
96    public static final String WAIT_ON_REGIONSERVERS_MAXTOSTART =
97        "hbase.master.wait.on.regionservers.maxtostart";
98  
99    public static final String WAIT_ON_REGIONSERVERS_MINTOSTART =
100       "hbase.master.wait.on.regionservers.mintostart";
101 
102   public static final String WAIT_ON_REGIONSERVERS_TIMEOUT =
103       "hbase.master.wait.on.regionservers.timeout";
104 
105   public static final String WAIT_ON_REGIONSERVERS_INTERVAL =
106       "hbase.master.wait.on.regionservers.interval";
107 
108   private static final Log LOG = LogFactory.getLog(ServerManager.class);
109 
110   // Set if we are to shutdown the cluster.
111   private volatile boolean clusterShutdown = false;
112 
113   private final SortedMap<byte[], Long> flushedSequenceIdByRegion =
114     new ConcurrentSkipListMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
115 
116   /** Map of registered servers to their current load */
117   private final ConcurrentHashMap<ServerName, ServerLoad> onlineServers =
118     new ConcurrentHashMap<ServerName, ServerLoad>();
119 
120   /**
121    * Map of admin interfaces per registered regionserver; these interfaces we use to control
122    * regionservers out on the cluster
123    */
124   private final Map<ServerName, AdminService.BlockingInterface> rsAdmins =
125     new HashMap<ServerName, AdminService.BlockingInterface>();
126 
127   /**
128    * List of region servers <ServerName> that should not get any more new
129    * regions.
130    */
131   private final ArrayList<ServerName> drainingServers =
132     new ArrayList<ServerName>();
133 
134   private final Server master;
135   private final MasterServices services;
136   private final HConnection connection;
137 
138   private final DeadServer deadservers = new DeadServer();
139 
140   private final long maxSkew;
141   private final long warningSkew;
142 
143   private final RetryCounterFactory pingRetryCounterFactory;
144 
145   /**
146    * Set of region servers which are dead but not processed immediately. If one
147    * server died before master enables ServerShutdownHandler, the server will be
148    * added to this set and will be processed through calling
149    * {@link ServerManager#processQueuedDeadServers()} by master.
150    * <p>
151    * A dead server is a server instance known to be dead, not listed in the /hbase/rs
152    * znode any more. It may have not been submitted to ServerShutdownHandler yet
153    * because the handler is not enabled.
154    * <p>
155    * A dead server, which has been submitted to ServerShutdownHandler while the
156    * handler is not enabled, is queued up.
157    * <p>
158    * So this is a set of region servers known to be dead but not submitted to
159    * ServerShutdownHander for processing yet.
160    */
161   private Set<ServerName> queuedDeadServers = new HashSet<ServerName>();
162 
163   /**
164    * Set of region servers which are dead and submitted to ServerShutdownHandler to process but not
165    * fully processed immediately.
166    * <p>
167    * If one server died before assignment manager finished the failover cleanup, the server will be
168    * added to this set and will be processed through calling
169    * {@link ServerManager#processQueuedDeadServers()} by assignment manager.
170    * <p>
171    * The Boolean value indicates whether log split is needed inside ServerShutdownHandler
172    * <p>
173    * ServerShutdownHandler processes a dead server submitted to the handler after the handler is
174    * enabled. It may not be able to complete the processing because meta is not yet online or master
175    * is currently in startup mode. In this case, the dead server will be parked in this set
176    * temporarily.
177    */
178   private Map<ServerName, Boolean> requeuedDeadServers
179     = new ConcurrentHashMap<ServerName, Boolean>();
180 
181   /** Listeners that are called on server events. */
182   private List<ServerListener> listeners = new CopyOnWriteArrayList<ServerListener>();
183 
184   /**
185    * Constructor.
186    * @param master
187    * @param services
188    * @throws ZooKeeperConnectionException
189    */
190   public ServerManager(final Server master, final MasterServices services)
191       throws IOException {
192     this(master, services, true);
193   }
194 
195   @SuppressWarnings("deprecation")
196   ServerManager(final Server master, final MasterServices services,
197       final boolean connect) throws IOException {
198     this.master = master;
199     this.services = services;
200     Configuration c = master.getConfiguration();
201     maxSkew = c.getLong("hbase.master.maxclockskew", 30000);
202     warningSkew = c.getLong("hbase.master.warningclockskew", 10000);
203     this.connection = connect ? HConnectionManager.getConnection(c) : null;
204     int pingMaxAttempts = Math.max(1, master.getConfiguration().getInt(
205       "hbase.master.maximum.ping.server.attempts", 10));
206     int pingSleepInterval = Math.max(1, master.getConfiguration().getInt(
207       "hbase.master.ping.server.retry.sleep.interval", 100));
208     this.pingRetryCounterFactory = new RetryCounterFactory(pingMaxAttempts, pingSleepInterval);
209   }
210 
211   /**
212    * Add the listener to the notification list.
213    * @param listener The ServerListener to register
214    */
215   public void registerListener(final ServerListener listener) {
216     this.listeners.add(listener);
217   }
218 
219   /**
220    * Remove the listener from the notification list.
221    * @param listener The ServerListener to unregister
222    */
223   public boolean unregisterListener(final ServerListener listener) {
224     return this.listeners.remove(listener);
225   }
226 
227   /**
228    * Let the server manager know a new regionserver has come online
229    * @param ia The remote address
230    * @param port The remote port
231    * @param serverStartcode
232    * @param serverCurrentTime The current time of the region server in ms
233    * @return The ServerName we know this server as.
234    * @throws IOException
235    */
236   ServerName regionServerStartup(final InetAddress ia, final int port,
237     final long serverStartcode, long serverCurrentTime)
238   throws IOException {
239     // Test for case where we get a region startup message from a regionserver
240     // that has been quickly restarted but whose znode expiration handler has
241     // not yet run, or from a server whose fail we are currently processing.
242     // Test its host+port combo is present in serverAddresstoServerInfo.  If it
243     // is, reject the server and trigger its expiration. The next time it comes
244     // in, it should have been removed from serverAddressToServerInfo and queued
245     // for processing by ProcessServerShutdown.
246     ServerName sn = ServerName.valueOf(ia.getHostName(), port, serverStartcode);
247     checkClockSkew(sn, serverCurrentTime);
248     checkIsDead(sn, "STARTUP");
249     if (!checkAndRecordNewServer(sn, ServerLoad.EMPTY_SERVERLOAD)) {
250       LOG.warn("THIS SHOULD NOT HAPPEN, RegionServerStartup"
251         + " could not record the server: " + sn);
252     }
253     return sn;
254   }
255 
256   /**
257    * Updates last flushed sequence Ids for the regions on server sn
258    * @param sn
259    * @param hsl
260    */
261   private void updateLastFlushedSequenceIds(ServerName sn, ServerLoad hsl) {
262     Map<byte[], RegionLoad> regionsLoad = hsl.getRegionsLoad();
263     for (Entry<byte[], RegionLoad> entry : regionsLoad.entrySet()) {
264       byte[] encodedRegionName = Bytes.toBytes(HRegionInfo.encodeRegionName(entry.getKey()));
265       Long existingValue = flushedSequenceIdByRegion.get(encodedRegionName);
266       long l = entry.getValue().getCompleteSequenceId();
267       if (existingValue != null) {
268         if (l != -1 && l < existingValue) {
269           LOG.warn("RegionServer " + sn +
270               " indicates a last flushed sequence id (" + entry.getValue() +
271               ") that is less than the previous last flushed sequence id (" +
272               existingValue + ") for region " +
273               Bytes.toString(entry.getKey()) + " Ignoring.");
274 
275           continue; // Don't let smaller sequence ids override greater sequence ids.
276         }
277       }
278       flushedSequenceIdByRegion.put(encodedRegionName, l);
279     }
280   }
281 
282   void regionServerReport(ServerName sn,
283       ServerLoad sl) throws YouAreDeadException {
284     checkIsDead(sn, "REPORT");
285     if (null == this.onlineServers.replace(sn, sl)) {
286       // Already have this host+port combo and its just different start code?
287       // Just let the server in. Presume master joining a running cluster.
288       // recordNewServer is what happens at the end of reportServerStartup.
289       // The only thing we are skipping is passing back to the regionserver
290       // the ServerName to use. Here we presume a master has already done
291       // that so we'll press on with whatever it gave us for ServerName.
292       if (!checkAndRecordNewServer(sn, sl)) {
293         LOG.info("RegionServerReport ignored, could not record the server: " + sn);
294         return; // Not recorded, so no need to move on
295       }
296     }
297     updateLastFlushedSequenceIds(sn, sl);
298   }
299 
300   /**
301    * Check is a server of same host and port already exists,
302    * if not, or the existed one got a smaller start code, record it.
303    *
304    * @param sn the server to check and record
305    * @param sl the server load on the server
306    * @return true if the server is recorded, otherwise, false
307    */
308   boolean checkAndRecordNewServer(
309       final ServerName serverName, final ServerLoad sl) {
310     ServerName existingServer = null;
311     synchronized (this.onlineServers) {
312       existingServer = findServerWithSameHostnamePortWithLock(serverName);
313       if (existingServer != null && (existingServer.getStartcode() > serverName.getStartcode())) {
314         LOG.info("Server serverName=" + serverName + " rejected; we already have "
315             + existingServer.toString() + " registered with same hostname and port");
316         return false;
317       }
318       recordNewServerWithLock(serverName, sl);
319     }
320 
321     // Tell our listeners that a server was added
322     if (!this.listeners.isEmpty()) {
323       for (ServerListener listener : this.listeners) {
324         listener.serverAdded(serverName);
325       }
326     }
327 
328     // Note that we assume that same ts means same server, and don't expire in that case.
329     //  TODO: ts can theoretically collide due to clock shifts, so this is a bit hacky.
330     if (existingServer != null && (existingServer.getStartcode() < serverName.getStartcode())) {
331       LOG.info("Triggering server recovery; existingServer " +
332           existingServer + " looks stale, new server:" + serverName);
333       expireServer(existingServer);
334     }
335     return true;
336   }
337 
338   /**
339    * Checks if the clock skew between the server and the master. If the clock skew exceeds the
340    * configured max, it will throw an exception; if it exceeds the configured warning threshold,
341    * it will log a warning but start normally.
342    * @param serverName Incoming servers's name
343    * @param serverCurrentTime
344    * @throws ClockOutOfSyncException if the skew exceeds the configured max value
345    */
346   private void checkClockSkew(final ServerName serverName, final long serverCurrentTime)
347   throws ClockOutOfSyncException {
348     long skew = Math.abs(System.currentTimeMillis() - serverCurrentTime);
349     if (skew > maxSkew) {
350       String message = "Server " + serverName + " has been " +
351         "rejected; Reported time is too far out of sync with master.  " +
352         "Time difference of " + skew + "ms > max allowed of " + maxSkew + "ms";
353       LOG.warn(message);
354       throw new ClockOutOfSyncException(message);
355     } else if (skew > warningSkew){
356       String message = "Reported time for server " + serverName + " is out of sync with master " +
357         "by " + skew + "ms. (Warning threshold is " + warningSkew + "ms; " +
358         "error threshold is " + maxSkew + "ms)";
359       LOG.warn(message);
360     }
361   }
362 
363   /**
364    * If this server is on the dead list, reject it with a YouAreDeadException.
365    * If it was dead but came back with a new start code, remove the old entry
366    * from the dead list.
367    * @param serverName
368    * @param what START or REPORT
369    * @throws org.apache.hadoop.hbase.YouAreDeadException
370    */
371   private void checkIsDead(final ServerName serverName, final String what)
372       throws YouAreDeadException {
373     if (this.deadservers.isDeadServer(serverName)) {
374       // host name, port and start code all match with existing one of the
375       // dead servers. So, this server must be dead.
376       String message = "Server " + what + " rejected; currently processing " +
377           serverName + " as dead server";
378       LOG.debug(message);
379       throw new YouAreDeadException(message);
380     }
381     // remove dead server with same hostname and port of newly checking in rs after master
382     // initialization.See HBASE-5916 for more information.
383     if ((this.services == null || ((HMaster) this.services).isInitialized())
384         && this.deadservers.cleanPreviousInstance(serverName)) {
385       // This server has now become alive after we marked it as dead.
386       // We removed it's previous entry from the dead list to reflect it.
387       LOG.debug(what + ":" + " Server " + serverName + " came back up," +
388           " removed it from the dead servers list");
389     }
390   }
391 
392   /**
393    * Assumes onlineServers is locked.
394    * @return ServerName with matching hostname and port.
395    */
396   private ServerName findServerWithSameHostnamePortWithLock(
397       final ServerName serverName) {
398     for (ServerName sn: this.onlineServers.keySet()) {
399       if (ServerName.isSameHostnameAndPort(serverName, sn)) return sn;
400     }
401     return null;
402   }
403 
404   /**
405    * Adds the onlineServers list. onlineServers should be locked.
406    * @param serverName The remote servers name.
407    * @param sl
408    * @return Server load from the removed server, if any.
409    */
410   @VisibleForTesting
411   void recordNewServerWithLock(final ServerName serverName, final ServerLoad sl) {
412     LOG.info("Registering server=" + serverName);
413     this.onlineServers.put(serverName, sl);
414     this.rsAdmins.remove(serverName);
415   }
416 
417   public long getLastFlushedSequenceId(byte[] encodedRegionName) {
418     long seqId = -1L;
419     if (flushedSequenceIdByRegion.containsKey(encodedRegionName)) {
420       seqId = flushedSequenceIdByRegion.get(encodedRegionName);
421     }
422     return seqId;
423   }
424 
425   /**
426    * @param serverName
427    * @return ServerLoad if serverName is known else null
428    */
429   public ServerLoad getLoad(final ServerName serverName) {
430     return this.onlineServers.get(serverName);
431   }
432 
433   /**
434    * Compute the average load across all region servers.
435    * Currently, this uses a very naive computation - just uses the number of
436    * regions being served, ignoring stats about number of requests.
437    * @return the average load
438    */
439   public double getAverageLoad() {
440     int totalLoad = 0;
441     int numServers = 0;
442     double averageLoad;
443     for (ServerLoad sl: this.onlineServers.values()) {
444         numServers++;
445         totalLoad += sl.getNumberOfRegions();
446     }
447     averageLoad = (double)totalLoad / (double)numServers;
448     return averageLoad;
449   }
450 
451   /** @return the count of active regionservers */
452   int countOfRegionServers() {
453     // Presumes onlineServers is a concurrent map
454     return this.onlineServers.size();
455   }
456 
457   /**
458    * @return Read-only map of servers to serverinfo
459    */
460   public Map<ServerName, ServerLoad> getOnlineServers() {
461     // Presumption is that iterating the returned Map is OK.
462     synchronized (this.onlineServers) {
463       return Collections.unmodifiableMap(this.onlineServers);
464     }
465   }
466 
467 
468   public DeadServer getDeadServers() {
469     return this.deadservers;
470   }
471 
472   /**
473    * Checks if any dead servers are currently in progress.
474    * @return true if any RS are being processed as dead, false if not
475    */
476   public boolean areDeadServersInProgress() {
477     return this.deadservers.areDeadServersInProgress();
478   }
479 
480   void letRegionServersShutdown() {
481     long previousLogTime = 0;
482     int onlineServersCt;
483     while ((onlineServersCt = onlineServers.size()) > 0) {
484 
485       if (System.currentTimeMillis() > (previousLogTime + 1000)) {
486         StringBuilder sb = new StringBuilder();
487         // It's ok here to not sync on onlineServers - merely logging
488         for (ServerName key : this.onlineServers.keySet()) {
489           if (sb.length() > 0) {
490             sb.append(", ");
491           }
492           sb.append(key);
493         }
494         LOG.info("Waiting on regionserver(s) to go down " + sb.toString());
495         previousLogTime = System.currentTimeMillis();
496       }
497 
498       synchronized (onlineServers) {
499         try {
500           if (onlineServersCt == onlineServers.size()) onlineServers.wait(100);
501         } catch (InterruptedException ignored) {
502           // continue
503         }
504       }
505     }
506   }
507 
508   /*
509    * Expire the passed server.  Add it to list of dead servers and queue a
510    * shutdown processing.
511    */
512   public synchronized void expireServer(final ServerName serverName) {
513     if (!services.isServerShutdownHandlerEnabled()) {
514       LOG.info("Master doesn't enable ServerShutdownHandler during initialization, "
515           + "delay expiring server " + serverName);
516       this.queuedDeadServers.add(serverName);
517       return;
518     }
519     if (this.deadservers.isDeadServer(serverName)) {
520       // TODO: Can this happen?  It shouldn't be online in this case?
521       LOG.warn("Expiration of " + serverName +
522           " but server shutdown already in progress");
523       return;
524     }
525     synchronized (onlineServers) {
526       if (!this.onlineServers.containsKey(serverName)) {
527         LOG.warn("Expiration of " + serverName + " but server not online");
528       }
529       // Remove the server from the known servers lists and update load info BUT
530       // add to deadservers first; do this so it'll show in dead servers list if
531       // not in online servers list.
532       this.deadservers.add(serverName);
533       this.onlineServers.remove(serverName);
534       onlineServers.notifyAll();
535     }
536     this.rsAdmins.remove(serverName);
537     // If cluster is going down, yes, servers are going to be expiring; don't
538     // process as a dead server
539     if (this.clusterShutdown) {
540       LOG.info("Cluster shutdown set; " + serverName +
541         " expired; onlineServers=" + this.onlineServers.size());
542       if (this.onlineServers.isEmpty()) {
543         master.stop("Cluster shutdown set; onlineServer=0");
544       }
545       return;
546     }
547 
548     boolean carryingMeta = services.getAssignmentManager().isCarryingMeta(serverName);
549     if (carryingMeta) {
550       this.services.getExecutorService().submit(new MetaServerShutdownHandler(this.master,
551         this.services, this.deadservers, serverName));
552     } else {
553       this.services.getExecutorService().submit(new ServerShutdownHandler(this.master,
554         this.services, this.deadservers, serverName, true));
555     }
556     LOG.debug("Added=" + serverName +
557       " to dead servers, submitted shutdown handler to be executed meta=" + carryingMeta);
558 
559     // Tell our listeners that a server was removed
560     if (!this.listeners.isEmpty()) {
561       for (ServerListener listener : this.listeners) {
562         listener.serverRemoved(serverName);
563       }
564     }
565   }
566 
567   public synchronized void processDeadServer(final ServerName serverName) {
568     this.processDeadServer(serverName, false);
569   }
570 
571   public synchronized void processDeadServer(final ServerName serverName, boolean shouldSplitHlog) {
572     // When assignment manager is cleaning up the zookeeper nodes and rebuilding the
573     // in-memory region states, region servers could be down. Meta table can and
574     // should be re-assigned, log splitting can be done too. However, it is better to
575     // wait till the cleanup is done before re-assigning user regions.
576     //
577     // We should not wait in the server shutdown handler thread since it can clog
578     // the handler threads and meta table could not be re-assigned in case
579     // the corresponding server is down. So we queue them up here instead.
580     if (!services.getAssignmentManager().isFailoverCleanupDone()) {
581       requeuedDeadServers.put(serverName, shouldSplitHlog);
582       return;
583     }
584 
585     this.deadservers.add(serverName);
586     this.services.getExecutorService().submit(
587       new ServerShutdownHandler(this.master, this.services, this.deadservers, serverName,
588           shouldSplitHlog));
589   }
590 
591   /**
592    * Process the servers which died during master's initialization. It will be
593    * called after HMaster#assignMeta and AssignmentManager#joinCluster.
594    * */
595   synchronized void processQueuedDeadServers() {
596     if (!services.isServerShutdownHandlerEnabled()) {
597       LOG.info("Master hasn't enabled ServerShutdownHandler");
598     }
599     Iterator<ServerName> serverIterator = queuedDeadServers.iterator();
600     while (serverIterator.hasNext()) {
601       ServerName tmpServerName = serverIterator.next();
602       expireServer(tmpServerName);
603       serverIterator.remove();
604       requeuedDeadServers.remove(tmpServerName);
605     }
606 
607     if (!services.getAssignmentManager().isFailoverCleanupDone()) {
608       LOG.info("AssignmentManager hasn't finished failover cleanup; waiting");
609     }
610 
611     for(ServerName tmpServerName : requeuedDeadServers.keySet()){
612       processDeadServer(tmpServerName, requeuedDeadServers.get(tmpServerName));
613     }
614     requeuedDeadServers.clear();
615   }
616 
617   /*
618    * Remove the server from the drain list.
619    */
620   public boolean removeServerFromDrainList(final ServerName sn) {
621     // Warn if the server (sn) is not online.  ServerName is of the form:
622     // <hostname> , <port> , <startcode>
623 
624     if (!this.isServerOnline(sn)) {
625       LOG.warn("Server " + sn + " is not currently online. " +
626                "Removing from draining list anyway, as requested.");
627     }
628     // Remove the server from the draining servers lists.
629     return this.drainingServers.remove(sn);
630   }
631 
632   /*
633    * Add the server to the drain list.
634    */
635   public boolean addServerToDrainList(final ServerName sn) {
636     // Warn if the server (sn) is not online.  ServerName is of the form:
637     // <hostname> , <port> , <startcode>
638 
639     if (!this.isServerOnline(sn)) {
640       LOG.warn("Server " + sn + " is not currently online. " +
641                "Ignoring request to add it to draining list.");
642       return false;
643     }
644     // Add the server to the draining servers lists, if it's not already in
645     // it.
646     if (this.drainingServers.contains(sn)) {
647       LOG.warn("Server " + sn + " is already in the draining server list." +
648                "Ignoring request to add it again.");
649       return false;
650     }
651     return this.drainingServers.add(sn);
652   }
653 
654   // RPC methods to region servers
655 
656   /**
657    * Sends an OPEN RPC to the specified server to open the specified region.
658    * <p>
659    * Open should not fail but can if server just crashed.
660    * <p>
661    * @param server server to open a region
662    * @param region region to open
663    * @param versionOfOfflineNode that needs to be present in the offline node
664    * when RS tries to change the state from OFFLINE to other states.
665    * @param favoredNodes
666    */
667   public RegionOpeningState sendRegionOpen(final ServerName server,
668       HRegionInfo region, int versionOfOfflineNode, List<ServerName> favoredNodes)
669   throws IOException {
670     AdminService.BlockingInterface admin = getRsAdmin(server);
671     if (admin == null) {
672       LOG.warn("Attempting to send OPEN RPC to server " + server.toString() +
673         " failed because no RPC connection found to this server");
674       return RegionOpeningState.FAILED_OPENING;
675     }
676     OpenRegionRequest request = RequestConverter.buildOpenRegionRequest(server, 
677       region, versionOfOfflineNode, favoredNodes, 
678       (RecoveryMode.LOG_REPLAY == this.services.getMasterFileSystem().getLogRecoveryMode()));
679     try {
680       OpenRegionResponse response = admin.openRegion(null, request);
681       return ResponseConverter.getRegionOpeningState(response);
682     } catch (ServiceException se) {
683       throw ProtobufUtil.getRemoteException(se);
684     }
685   }
686 
687   /**
688    * Sends an OPEN RPC to the specified server to open the specified region.
689    * <p>
690    * Open should not fail but can if server just crashed.
691    * <p>
692    * @param server server to open a region
693    * @param regionOpenInfos info of a list of regions to open
694    * @return a list of region opening states
695    */
696   public List<RegionOpeningState> sendRegionOpen(ServerName server,
697       List<Triple<HRegionInfo, Integer, List<ServerName>>> regionOpenInfos)
698   throws IOException {
699     AdminService.BlockingInterface admin = getRsAdmin(server);
700     if (admin == null) {
701       LOG.warn("Attempting to send OPEN RPC to server " + server.toString() +
702         " failed because no RPC connection found to this server");
703       return null;
704     }
705 
706     OpenRegionRequest request = RequestConverter.buildOpenRegionRequest(server, regionOpenInfos,
707       (RecoveryMode.LOG_REPLAY == this.services.getMasterFileSystem().getLogRecoveryMode()));
708     try {
709       OpenRegionResponse response = admin.openRegion(null, request);
710       return ResponseConverter.getRegionOpeningStateList(response);
711     } catch (ServiceException se) {
712       throw ProtobufUtil.getRemoteException(se);
713     }
714   }
715 
716   /**
717    * Sends an CLOSE RPC to the specified server to close the specified region.
718    * <p>
719    * A region server could reject the close request because it either does not
720    * have the specified region or the region is being split.
721    * @param server server to open a region
722    * @param region region to open
723    * @param versionOfClosingNode
724    *   the version of znode to compare when RS transitions the znode from
725    *   CLOSING state.
726    * @param dest - if the region is moved to another server, the destination server. null otherwise.
727    * @return true if server acknowledged close, false if not
728    * @throws IOException
729    */
730   public boolean sendRegionClose(ServerName server, HRegionInfo region,
731     int versionOfClosingNode, ServerName dest, boolean transitionInZK) throws IOException {
732     if (server == null) throw new NullPointerException("Passed server is null");
733     AdminService.BlockingInterface admin = getRsAdmin(server);
734     if (admin == null) {
735       throw new IOException("Attempting to send CLOSE RPC to server " +
736         server.toString() + " for region " +
737         region.getRegionNameAsString() +
738         " failed because no RPC connection found to this server");
739     }
740     return ProtobufUtil.closeRegion(admin, server, region.getRegionName(),
741       versionOfClosingNode, dest, transitionInZK);
742   }
743 
744   public boolean sendRegionClose(ServerName server,
745       HRegionInfo region, int versionOfClosingNode) throws IOException {
746     return sendRegionClose(server, region, versionOfClosingNode, null, true);
747   }
748 
749   /**
750    * Sends an MERGE REGIONS RPC to the specified server to merge the specified
751    * regions.
752    * <p>
753    * A region server could reject the close request because it either does not
754    * have the specified region.
755    * @param server server to merge regions
756    * @param region_a region to merge
757    * @param region_b region to merge
758    * @param forcible true if do a compulsory merge, otherwise we will only merge
759    *          two adjacent regions
760    * @throws IOException
761    */
762   public void sendRegionsMerge(ServerName server, HRegionInfo region_a,
763       HRegionInfo region_b, boolean forcible) throws IOException {
764     if (server == null)
765       throw new NullPointerException("Passed server is null");
766     if (region_a == null || region_b == null)
767       throw new NullPointerException("Passed region is null");
768     AdminService.BlockingInterface admin = getRsAdmin(server);
769     if (admin == null) {
770       throw new IOException("Attempting to send MERGE REGIONS RPC to server "
771           + server.toString() + " for region "
772           + region_a.getRegionNameAsString() + ","
773           + region_b.getRegionNameAsString()
774           + " failed because no RPC connection found to this server");
775     }
776     ProtobufUtil.mergeRegions(admin, region_a, region_b, forcible);
777   }
778 
779   /**
780    * Check if a region server is reachable and has the expected start code
781    */
782   public boolean isServerReachable(ServerName server) {
783     if (server == null) throw new NullPointerException("Passed server is null");
784 
785     RetryCounter retryCounter = pingRetryCounterFactory.create();
786     while (retryCounter.shouldRetry()) {
787       try {
788         AdminService.BlockingInterface admin = getRsAdmin(server);
789         if (admin != null) {
790           ServerInfo info = ProtobufUtil.getServerInfo(admin);
791           return info != null && info.hasServerName()
792             && server.getStartcode() == info.getServerName().getStartCode();
793         }
794       } catch (IOException ioe) {
795         LOG.debug("Couldn't reach " + server + ", try=" + retryCounter.getAttemptTimes()
796           + " of " + retryCounter.getMaxAttempts(), ioe);
797         try {
798           retryCounter.sleepUntilNextRetry();
799         } catch(InterruptedException ie) {
800           Thread.currentThread().interrupt();
801         }
802       }
803     }
804     return false;
805   }
806 
807     /**
808     * @param sn
809     * @return Admin interface for the remote regionserver named <code>sn</code>
810     * @throws IOException
811     * @throws RetriesExhaustedException wrapping a ConnectException if failed
812     */
813   private AdminService.BlockingInterface getRsAdmin(final ServerName sn)
814   throws IOException {
815     AdminService.BlockingInterface admin = this.rsAdmins.get(sn);
816     if (admin == null) {
817       LOG.debug("New admin connection to " + sn.toString());
818       admin = this.connection.getAdmin(sn);
819       this.rsAdmins.put(sn, admin);
820     }
821     return admin;
822   }
823 
824   /**
825    * Wait for the region servers to report in.
826    * We will wait until one of this condition is met:
827    *  - the master is stopped
828    *  - the 'hbase.master.wait.on.regionservers.maxtostart' number of
829    *    region servers is reached
830    *  - the 'hbase.master.wait.on.regionservers.mintostart' is reached AND
831    *   there have been no new region server in for
832    *      'hbase.master.wait.on.regionservers.interval' time AND
833    *   the 'hbase.master.wait.on.regionservers.timeout' is reached
834    *
835    * @throws InterruptedException
836    */
837   public void waitForRegionServers(MonitoredTask status)
838   throws InterruptedException {
839     final long interval = this.master.getConfiguration().
840       getLong(WAIT_ON_REGIONSERVERS_INTERVAL, 1500);
841     final long timeout = this.master.getConfiguration().
842       getLong(WAIT_ON_REGIONSERVERS_TIMEOUT, 4500);
843     int minToStart = this.master.getConfiguration().
844       getInt(WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
845     if (minToStart < 1) {
846       LOG.warn(String.format(
847         "The value of '%s' (%d) can not be less than 1, ignoring.",
848         WAIT_ON_REGIONSERVERS_MINTOSTART, minToStart));
849       minToStart = 1;
850     }
851     int maxToStart = this.master.getConfiguration().
852       getInt(WAIT_ON_REGIONSERVERS_MAXTOSTART, Integer.MAX_VALUE);
853     if (maxToStart < minToStart) {
854         LOG.warn(String.format(
855             "The value of '%s' (%d) is set less than '%s' (%d), ignoring.",
856             WAIT_ON_REGIONSERVERS_MAXTOSTART, maxToStart,
857             WAIT_ON_REGIONSERVERS_MINTOSTART, minToStart));
858         maxToStart = Integer.MAX_VALUE;
859     }
860 
861     long now =  System.currentTimeMillis();
862     final long startTime = now;
863     long slept = 0;
864     long lastLogTime = 0;
865     long lastCountChange = startTime;
866     int count = countOfRegionServers();
867     int oldCount = 0;
868     while (
869       !this.master.isStopped() &&
870         count < maxToStart &&
871         (lastCountChange+interval > now || timeout > slept || count < minToStart)
872       ){
873 
874       // Log some info at every interval time or if there is a change
875       if (oldCount != count || lastLogTime+interval < now){
876         lastLogTime = now;
877         String msg =
878           "Waiting for region servers count to settle; currently"+
879             " checked in " + count + ", slept for " + slept + " ms," +
880             " expecting minimum of " + minToStart + ", maximum of "+ maxToStart+
881             ", timeout of "+timeout+" ms, interval of "+interval+" ms.";
882         LOG.info(msg);
883         status.setStatus(msg);
884       }
885 
886       // We sleep for some time
887       final long sleepTime = 50;
888       Thread.sleep(sleepTime);
889       now =  System.currentTimeMillis();
890       slept = now - startTime;
891 
892       oldCount = count;
893       count = countOfRegionServers();
894       if (count != oldCount) {
895         lastCountChange = now;
896       }
897     }
898 
899     LOG.info("Finished waiting for region servers count to settle;" +
900       " checked in " + count + ", slept for " + slept + " ms," +
901       " expecting minimum of " + minToStart + ", maximum of "+ maxToStart+","+
902       " master is "+ (this.master.isStopped() ? "stopped.": "running.")
903     );
904   }
905 
906   /**
907    * @return A copy of the internal list of online servers.
908    */
909   public List<ServerName> getOnlineServersList() {
910     // TODO: optimize the load balancer call so we don't need to make a new list
911     // TODO: FIX. THIS IS POPULAR CALL.
912     return new ArrayList<ServerName>(this.onlineServers.keySet());
913   }
914 
915   /**
916    * @return A copy of the internal list of draining servers.
917    */
918   public List<ServerName> getDrainingServersList() {
919     return new ArrayList<ServerName>(this.drainingServers);
920   }
921 
922   /**
923    * @return A copy of the internal set of deadNotExpired servers.
924    */
925   Set<ServerName> getDeadNotExpiredServers() {
926     return new HashSet<ServerName>(this.queuedDeadServers);
927   }
928 
929   /**
930    * During startup, if we figure it is not a failover, i.e. there is
931    * no more HLog files to split, we won't try to recover these dead servers.
932    * So we just remove them from the queue. Use caution in calling this.
933    */
934   void removeRequeuedDeadServers() {
935     requeuedDeadServers.clear();
936   }
937 
938   /**
939    * @return A copy of the internal map of requeuedDeadServers servers and their corresponding
940    *         splitlog need flag.
941    */
942   Map<ServerName, Boolean> getRequeuedDeadServers() {
943     return Collections.unmodifiableMap(this.requeuedDeadServers);
944   }
945 
946   public boolean isServerOnline(ServerName serverName) {
947     return serverName != null && onlineServers.containsKey(serverName);
948   }
949 
950   /**
951    * Check if a server is known to be dead.  A server can be online,
952    * or known to be dead, or unknown to this manager (i.e, not online,
953    * not known to be dead either. it is simply not tracked by the
954    * master any more, for example, a very old previous instance).
955    */
956   public synchronized boolean isServerDead(ServerName serverName) {
957     return serverName == null || deadservers.isDeadServer(serverName)
958       || queuedDeadServers.contains(serverName)
959       || requeuedDeadServers.containsKey(serverName);
960   }
961 
962   public void shutdownCluster() {
963     this.clusterShutdown = true;
964     this.master.stop("Cluster shutdown requested");
965   }
966 
967   public boolean isClusterShutdown() {
968     return this.clusterShutdown;
969   }
970 
971   /**
972    * Stop the ServerManager.  Currently closes the connection to the master.
973    */
974   public void stop() {
975     if (connection != null) {
976       try {
977         connection.close();
978       } catch (IOException e) {
979         LOG.error("Attempt to close connection to master failed", e);
980       }
981     }
982   }
983 
984   /**
985    * Creates a list of possible destinations for a region. It contains the online servers, but not
986    *  the draining or dying servers.
987    *  @param serverToExclude can be null if there is no server to exclude
988    */
989   public List<ServerName> createDestinationServersList(final ServerName serverToExclude){
990     final List<ServerName> destServers = getOnlineServersList();
991 
992     if (serverToExclude != null){
993       destServers.remove(serverToExclude);
994     }
995 
996     // Loop through the draining server list and remove them from the server list
997     final List<ServerName> drainingServersCopy = getDrainingServersList();
998     if (!drainingServersCopy.isEmpty()) {
999       for (final ServerName server: drainingServersCopy) {
1000         destServers.remove(server);
1001       }
1002     }
1003 
1004     // Remove the deadNotExpired servers from the server list.
1005     removeDeadNotExpiredServers(destServers);
1006 
1007     return destServers;
1008   }
1009 
1010   /**
1011    * Calls {@link #createDestinationServersList} without server to exclude.
1012    */
1013   public List<ServerName> createDestinationServersList(){
1014     return createDestinationServersList(null);
1015   }
1016 
1017     /**
1018     * Loop through the deadNotExpired server list and remove them from the
1019     * servers.
1020     * This function should be used carefully outside of this class. You should use a high level
1021     *  method such as {@link #createDestinationServersList()} instead of managing you own list.
1022     */
1023   void removeDeadNotExpiredServers(List<ServerName> servers) {
1024     Set<ServerName> deadNotExpiredServersCopy = this.getDeadNotExpiredServers();
1025     if (!deadNotExpiredServersCopy.isEmpty()) {
1026       for (ServerName server : deadNotExpiredServersCopy) {
1027         LOG.debug("Removing dead but not expired server: " + server
1028           + " from eligible server pool.");
1029         servers.remove(server);
1030       }
1031     }
1032   }
1033 
1034   /**
1035    * To clear any dead server with same host name and port of any online server
1036    */
1037   void clearDeadServersWithSameHostNameAndPortOfOnlineServer() {
1038     for (ServerName serverName : getOnlineServersList()) {
1039       deadservers.cleanAllPreviousInstances(serverName);
1040     }
1041   }
1042 }