View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  
22  import java.io.Closeable;
23  import java.io.IOException;
24  import java.io.InterruptedIOException;
25  import java.net.SocketTimeoutException;
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.LinkedList;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.concurrent.atomic.AtomicInteger;
32  import java.util.concurrent.atomic.AtomicReference;
33  import java.util.regex.Pattern;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.hbase.Abortable;
39  import org.apache.hadoop.hbase.ClusterStatus;
40  import org.apache.hadoop.hbase.HBaseConfiguration;
41  import org.apache.hadoop.hbase.HBaseIOException;
42  import org.apache.hadoop.hbase.HColumnDescriptor;
43  import org.apache.hadoop.hbase.HConstants;
44  import org.apache.hadoop.hbase.HRegionInfo;
45  import org.apache.hadoop.hbase.HRegionLocation;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.MasterNotRunningException;
48  import org.apache.hadoop.hbase.NamespaceDescriptor;
49  import org.apache.hadoop.hbase.NotServingRegionException;
50  import org.apache.hadoop.hbase.RegionException;
51  import org.apache.hadoop.hbase.ServerName;
52  import org.apache.hadoop.hbase.TableExistsException;
53  import org.apache.hadoop.hbase.TableName;
54  import org.apache.hadoop.hbase.TableNotDisabledException;
55  import org.apache.hadoop.hbase.TableNotEnabledException;
56  import org.apache.hadoop.hbase.TableNotFoundException;
57  import org.apache.hadoop.hbase.UnknownRegionException;
58  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
59  import org.apache.hadoop.hbase.catalog.CatalogTracker;
60  import org.apache.hadoop.hbase.catalog.MetaReader;
61  import org.apache.hadoop.hbase.classification.InterfaceAudience;
62  import org.apache.hadoop.hbase.classification.InterfaceStability;
63  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
64  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
65  import org.apache.hadoop.hbase.exceptions.DeserializationException;
66  import org.apache.hadoop.hbase.exceptions.MergeRegionException;
67  import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
68  import org.apache.hadoop.hbase.ipc.MasterCoprocessorRpcChannel;
69  import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
70  import org.apache.hadoop.hbase.ipc.RegionServerCoprocessorRpcChannel;
71  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
72  import org.apache.hadoop.hbase.protobuf.RequestConverter;
73  import org.apache.hadoop.hbase.protobuf.ResponseConverter;
74  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
75  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
76  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
77  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest;
78  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionRequest;
79  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
80  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
81  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
82  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest;
83  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterResponse;
84  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerRequest;
85  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
86  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
87  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
88  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
89  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
90  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription;
91  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
92  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema;
93  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnRequest;
94  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AssignRegionRequest;
95  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateNamespaceRequest;
96  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest;
97  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteColumnRequest;
98  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteNamespaceRequest;
99  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteSnapshotRequest;
100 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteTableRequest;
101 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableRequest;
102 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsRequest;
103 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableRequest;
104 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ExecProcedureRequest;
105 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ExecProcedureResponse;
106 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetClusterStatusRequest;
107 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetCompletedSnapshotsRequest;
108 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetNamespaceDescriptorRequest;
109 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusRequest;
110 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusResponse;
111 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsRequest;
112 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
113 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneRequest;
114 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneResponse;
115 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneRequest;
116 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneResponse;
117 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
118 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
119 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListNamespaceDescriptorsRequest;
120 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableDescriptorsByNamespaceRequest;
121 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableNamesByNamespaceRequest;
122 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyColumnRequest;
123 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyNamespaceRequest;
124 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableRequest;
125 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionRequest;
126 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotRequest;
127 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotResponse;
128 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningRequest;
129 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownRequest;
130 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotRequest;
131 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotResponse;
132 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterRequest;
133 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.TruncateTableRequest;
134 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.UnassignRegionRequest;
135 import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
136 import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
137 import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
138 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
139 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
140 import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
141 import org.apache.hadoop.hbase.util.Addressing;
142 import org.apache.hadoop.hbase.util.Bytes;
143 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
144 import org.apache.hadoop.hbase.util.Pair;
145 import org.apache.hadoop.ipc.RemoteException;
146 import org.apache.hadoop.util.StringUtils;
147 import org.apache.zookeeper.KeeperException;
148 
149 import com.google.common.annotations.VisibleForTesting;
150 import com.google.protobuf.ByteString;
151 import com.google.protobuf.ServiceException;
152 
153 /**
154  * Provides an interface to manage HBase database table metadata + general
155  * administrative functions.  Use HBaseAdmin to create, drop, list, enable and
156  * disable tables. Use it also to add and drop table column families.
157  *
158  * <p>See {@link HTable} to add, update, and delete data from an individual table.
159  * <p>Currently HBaseAdmin instances are not expected to be long-lived.  For
160  * example, an HBaseAdmin instance will not ride over a Master restart.
161  */
162 @InterfaceAudience.Public
163 @InterfaceStability.Evolving
164 public class HBaseAdmin implements Abortable, Closeable {
165   private static final Log LOG = LogFactory.getLog(HBaseAdmin.class);
166 
167   // We use the implementation class rather then the interface because we
168   //  need the package protected functions to get the connection to master
169   private HConnection connection;
170 
171   private volatile Configuration conf;
172   private final long pause;
173   private final int numRetries;
174   // Some operations can take a long time such as disable of big table.
175   // numRetries is for 'normal' stuff... Multiply by this factor when
176   // want to wait a long time.
177   private final int retryLongerMultiplier;
178   private boolean aborted;
179   private boolean cleanupConnectionOnClose = false; // close the connection in close()
180   private boolean closed = false;
181 
182   private RpcRetryingCallerFactory rpcCallerFactory;
183 
184   /**
185    * Constructor.
186    * See {@link #HBaseAdmin(HConnection connection)}
187    *
188    * @param c Configuration object. Copied internally.
189    */
190   public HBaseAdmin(Configuration c)
191   throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
192     // Will not leak connections, as the new implementation of the constructor
193     // does not throw exceptions anymore.
194     this(HConnectionManager.getConnection(new Configuration(c)));
195     this.cleanupConnectionOnClose = true;
196   }
197 
198  /**
199   * Constructor for externally managed HConnections.
200   * The connection to master will be created when required by admin functions.
201   *
202   * @param connection The HConnection instance to use
203   * @throws MasterNotRunningException, ZooKeeperConnectionException are not
204   *  thrown anymore but kept into the interface for backward api compatibility
205   */
206   public HBaseAdmin(HConnection connection)
207       throws MasterNotRunningException, ZooKeeperConnectionException {
208     this.conf = connection.getConfiguration();
209     this.connection = connection;
210 
211     this.pause = this.conf.getLong(HConstants.HBASE_CLIENT_PAUSE,
212         HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
213     this.numRetries = this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
214         HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
215     this.retryLongerMultiplier = this.conf.getInt(
216         "hbase.client.retries.longer.multiplier", 10);
217     this.rpcCallerFactory = RpcRetryingCallerFactory.instantiate(this.conf,
218       connection.getStatisticsTracker());
219   }
220 
221   /**
222    * @return A new CatalogTracker instance; call {@link #cleanupCatalogTracker(CatalogTracker)}
223    * to cleanup the returned catalog tracker.
224    * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException
225    * @throws IOException
226    * @see #cleanupCatalogTracker(CatalogTracker)
227    */
228   @VisibleForTesting
229   synchronized CatalogTracker getCatalogTracker()
230   throws ZooKeeperConnectionException, IOException {
231     boolean succeeded = false;
232     CatalogTracker ct = null;
233     try {
234       ct = new CatalogTracker(this.conf);
235       startCatalogTracker(ct);
236       succeeded = true;
237     } catch (InterruptedException e) {
238       // Let it out as an IOE for now until we redo all so tolerate IEs
239       throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
240     } finally {
241       // If we did not succeed but created a catalogtracker, clean it up. CT has a ZK instance
242       // in it and we'll leak if we don't do the 'stop'.
243       if (!succeeded && ct != null) {
244         try {
245           ct.stop();
246         } catch (RuntimeException re) {
247           LOG.error("Failed to clean up HBase's internal catalog tracker after a failed initialization. " +
248             "We may have leaked network connections to ZooKeeper; they won't be cleaned up until " +
249             "the JVM exits. If you see a large number of stale connections to ZooKeeper this is likely " +
250             "the cause. The following exception details will be needed for assistance from the " +
251             "HBase community.", re);
252         }
253         ct = null;
254       }
255     }
256     return ct;
257   }
258 
259   @VisibleForTesting
260   CatalogTracker startCatalogTracker(final CatalogTracker ct)
261   throws IOException, InterruptedException {
262     ct.start();
263     return ct;
264   }
265 
266   @VisibleForTesting
267   void cleanupCatalogTracker(final CatalogTracker ct) {
268     ct.stop();
269   }
270 
271   @Override
272   public void abort(String why, Throwable e) {
273     // Currently does nothing but throw the passed message and exception
274     this.aborted = true;
275     throw new RuntimeException(why, e);
276   }
277 
278   @Override
279   public boolean isAborted(){
280     return this.aborted;
281   }
282 
283   /** @return HConnection used by this object. */
284   public HConnection getConnection() {
285     return connection;
286   }
287 
288   /** @return - true if the master server is running. Throws an exception
289    *  otherwise.
290    * @throws ZooKeeperConnectionException
291    * @throws MasterNotRunningException
292    */
293   public boolean isMasterRunning()
294   throws MasterNotRunningException, ZooKeeperConnectionException {
295     return connection.isMasterRunning();
296   }
297 
298   /**
299    * @param tableName Table to check.
300    * @return True if table exists already.
301    * @throws IOException
302    */
303   public boolean tableExists(final TableName tableName)
304   throws IOException {
305     boolean b = false;
306     CatalogTracker ct = getCatalogTracker();
307     try {
308       b = MetaReader.tableExists(ct, tableName);
309     } finally {
310       cleanupCatalogTracker(ct);
311     }
312     return b;
313   }
314 
315   public boolean tableExists(final byte[] tableName)
316   throws IOException {
317     return tableExists(TableName.valueOf(tableName));
318   }
319 
320   public boolean tableExists(final String tableName)
321   throws IOException {
322     return tableExists(TableName.valueOf(tableName));
323   }
324 
325   /**
326    * List all the userspace tables.  In other words, scan the hbase:meta table.
327    *
328    * If we wanted this to be really fast, we could implement a special
329    * catalog table that just contains table names and their descriptors.
330    * Right now, it only exists as part of the hbase:meta table's region info.
331    *
332    * @return - returns an array of HTableDescriptors
333    * @throws IOException if a remote or network exception occurs
334    */
335   public HTableDescriptor[] listTables() throws IOException {
336     return this.connection.listTables();
337   }
338 
339   /**
340    * List all the userspace tables matching the given pattern.
341    *
342    * @param pattern The compiled regular expression to match against
343    * @return - returns an array of HTableDescriptors
344    * @throws IOException if a remote or network exception occurs
345    * @see #listTables()
346    */
347   public HTableDescriptor[] listTables(Pattern pattern) throws IOException {
348     List<HTableDescriptor> matched = new LinkedList<HTableDescriptor>();
349     HTableDescriptor[] tables = listTables();
350     for (HTableDescriptor table : tables) {
351       if (pattern.matcher(table.getTableName().getNameAsString()).matches()) {
352         matched.add(table);
353       }
354     }
355     return matched.toArray(new HTableDescriptor[matched.size()]);
356   }
357 
358   /**
359    * List all the userspace tables matching the given regular expression.
360    *
361    * @param regex The regular expression to match against
362    * @return - returns an array of HTableDescriptors
363    * @throws IOException if a remote or network exception occurs
364    * @see #listTables(java.util.regex.Pattern)
365    */
366   public HTableDescriptor[] listTables(String regex) throws IOException {
367     return listTables(Pattern.compile(regex));
368   }
369 
370   /**
371    * List all of the names of userspace tables.
372    * @return String[] table names
373    * @throws IOException if a remote or network exception occurs
374    */
375   @Deprecated
376   public String[] getTableNames() throws IOException {
377     return this.connection.getTableNames();
378   }
379 
380   /**
381    * List all of the names of userspace tables matching the given regular expression.
382    * @param pattern The regular expression to match against
383    * @return String[] table names
384    * @throws IOException if a remote or network exception occurs
385    */
386   @Deprecated
387   public String[] getTableNames(Pattern pattern) throws IOException {
388     List<String> matched = new ArrayList<String>();
389     for (String name: this.connection.getTableNames()) {
390       if (pattern.matcher(name).matches()) {
391         matched.add(name);
392       }
393     }
394     return matched.toArray(new String[matched.size()]);
395   }
396 
397   /**
398    * List all of the names of userspace tables matching the given regular expression.
399    * @param regex The regular expression to match against
400    * @return String[] table names
401    * @throws IOException if a remote or network exception occurs
402    */
403   @Deprecated
404   public String[] getTableNames(String regex) throws IOException {
405     return getTableNames(Pattern.compile(regex));
406   }
407 
408   /**
409    * List all of the names of userspace tables.
410    * @return TableName[] table names
411    * @throws IOException if a remote or network exception occurs
412    */
413   public TableName[] listTableNames() throws IOException {
414     return this.connection.listTableNames();
415   }
416 
417   /**
418    * Method for getting the tableDescriptor
419    * @param tableName as a byte []
420    * @return the tableDescriptor
421    * @throws TableNotFoundException
422    * @throws IOException if a remote or network exception occurs
423    */
424   public HTableDescriptor getTableDescriptor(final TableName tableName)
425   throws TableNotFoundException, IOException {
426     return this.connection.getHTableDescriptor(tableName);
427   }
428 
429   public HTableDescriptor getTableDescriptor(final byte[] tableName)
430   throws TableNotFoundException, IOException {
431     return getTableDescriptor(TableName.valueOf(tableName));
432   }
433 
434   private long getPauseTime(int tries) {
435     int triesCount = tries;
436     if (triesCount >= HConstants.RETRY_BACKOFF.length) {
437       triesCount = HConstants.RETRY_BACKOFF.length - 1;
438     }
439     return this.pause * HConstants.RETRY_BACKOFF[triesCount];
440   }
441 
442   /**
443    * Creates a new table.
444    * Synchronous operation.
445    *
446    * @param desc table descriptor for table
447    *
448    * @throws IllegalArgumentException if the table name is reserved
449    * @throws MasterNotRunningException if master is not running
450    * @throws TableExistsException if table already exists (If concurrent
451    * threads, the table may have been created between test-for-existence
452    * and attempt-at-creation).
453    * @throws IOException if a remote or network exception occurs
454    */
455   public void createTable(HTableDescriptor desc)
456   throws IOException {
457     createTable(desc, null);
458   }
459 
460   /**
461    * Creates a new table with the specified number of regions.  The start key
462    * specified will become the end key of the first region of the table, and
463    * the end key specified will become the start key of the last region of the
464    * table (the first region has a null start key and the last region has a
465    * null end key).
466    *
467    * BigInteger math will be used to divide the key range specified into
468    * enough segments to make the required number of total regions.
469    *
470    * Synchronous operation.
471    *
472    * @param desc table descriptor for table
473    * @param startKey beginning of key range
474    * @param endKey end of key range
475    * @param numRegions the total number of regions to create
476    *
477    * @throws IllegalArgumentException if the table name is reserved
478    * @throws MasterNotRunningException if master is not running
479    * @throws org.apache.hadoop.hbase.TableExistsException if table already exists (If concurrent
480    * threads, the table may have been created between test-for-existence
481    * and attempt-at-creation).
482    * @throws IOException
483    */
484   public void createTable(HTableDescriptor desc, byte [] startKey,
485       byte [] endKey, int numRegions)
486   throws IOException {
487     if(numRegions < 3) {
488       throw new IllegalArgumentException("Must create at least three regions");
489     } else if(Bytes.compareTo(startKey, endKey) >= 0) {
490       throw new IllegalArgumentException("Start key must be smaller than end key");
491     }
492     if (numRegions == 3) {
493       createTable(desc, new byte[][]{startKey, endKey});
494       return;
495     }
496     byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
497     if(splitKeys == null || splitKeys.length != numRegions - 1) {
498       throw new IllegalArgumentException("Unable to split key range into enough regions");
499     }
500     createTable(desc, splitKeys);
501   }
502 
503   /**
504    * Creates a new table with an initial set of empty regions defined by the
505    * specified split keys.  The total number of regions created will be the
506    * number of split keys plus one. Synchronous operation.
507    * Note : Avoid passing empty split key.
508    *
509    * @param desc table descriptor for table
510    * @param splitKeys array of split keys for the initial regions of the table
511    *
512    * @throws IllegalArgumentException if the table name is reserved, if the split keys
513    * are repeated and if the split key has empty byte array.
514    * @throws MasterNotRunningException if master is not running
515    * @throws org.apache.hadoop.hbase.TableExistsException if table already exists (If concurrent
516    * threads, the table may have been created between test-for-existence
517    * and attempt-at-creation).
518    * @throws IOException
519    */
520   public void createTable(final HTableDescriptor desc, byte [][] splitKeys)
521   throws IOException {
522     try {
523       createTableAsync(desc, splitKeys);
524     } catch (SocketTimeoutException ste) {
525       LOG.warn("Creating " + desc.getTableName() + " took too long", ste);
526     }
527     int numRegs = splitKeys == null ? 1 : splitKeys.length + 1;
528     int prevRegCount = 0;
529     boolean doneWithMetaScan = false;
530     for (int tries = 0; tries < this.numRetries * this.retryLongerMultiplier;
531       ++tries) {
532       if (!doneWithMetaScan) {
533         // Wait for new table to come on-line
534         final AtomicInteger actualRegCount = new AtomicInteger(0);
535         MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
536           @Override
537           public boolean processRow(Result rowResult) throws IOException {
538             HRegionInfo info = HRegionInfo.getHRegionInfo(rowResult);
539             if (info == null) {
540               LOG.warn("No serialized HRegionInfo in " + rowResult);
541               return true;
542             }
543             if (!info.getTable().equals(desc.getTableName())) {
544               return false;
545             }
546             ServerName serverName = HRegionInfo.getServerName(rowResult);
547             // Make sure that regions are assigned to server
548             if (!(info.isOffline() || info.isSplit()) && serverName != null
549                 && serverName.getHostAndPort() != null) {
550               actualRegCount.incrementAndGet();
551             }
552             return true;
553           }
554         };
555         MetaScanner.metaScan(conf, connection, visitor, desc.getTableName());
556         if (actualRegCount.get() < numRegs) {
557           if (tries == this.numRetries * this.retryLongerMultiplier - 1) {
558             throw new RegionOfflineException("Only " + actualRegCount.get() +
559               " of " + numRegs + " regions are online; retries exhausted.");
560           }
561           try { // Sleep
562             Thread.sleep(getPauseTime(tries));
563           } catch (InterruptedException e) {
564             throw new InterruptedIOException("Interrupted when opening" +
565               " regions; " + actualRegCount.get() + " of " + numRegs +
566               " regions processed so far");
567           }
568           if (actualRegCount.get() > prevRegCount) { // Making progress
569             prevRegCount = actualRegCount.get();
570             tries = -1;
571           }
572         } else {
573           doneWithMetaScan = true;
574           tries = -1;
575         }
576       } else if (isTableEnabled(desc.getTableName())) {
577         return;
578       } else {
579         try { // Sleep
580           Thread.sleep(getPauseTime(tries));
581         } catch (InterruptedException e) {
582           throw new InterruptedIOException("Interrupted when waiting" +
583             " for table to be enabled; meta scan was done");
584         }
585       }
586     }
587     throw new TableNotEnabledException(
588       "Retries exhausted while still waiting for table: "
589       + desc.getTableName() + " to be enabled");
590   }
591 
592   /**
593    * Creates a new table but does not block and wait for it to come online.
594    * Asynchronous operation.  To check if the table exists, use
595    * {@link #isTableAvailable} -- it is not safe to create an HTable
596    * instance to this table before it is available.
597    * Note : Avoid passing empty split key.
598    * @param desc table descriptor for table
599    *
600    * @throws IllegalArgumentException Bad table name, if the split keys
601    * are repeated and if the split key has empty byte array.
602    * @throws MasterNotRunningException if master is not running
603    * @throws org.apache.hadoop.hbase.TableExistsException if table already exists (If concurrent
604    * threads, the table may have been created between test-for-existence
605    * and attempt-at-creation).
606    * @throws IOException
607    */
608   public void createTableAsync(
609     final HTableDescriptor desc, final byte [][] splitKeys)
610   throws IOException {
611     if(desc.getTableName() == null) {
612       throw new IllegalArgumentException("TableName cannot be null");
613     }
614     if(splitKeys != null && splitKeys.length > 0) {
615       Arrays.sort(splitKeys, Bytes.BYTES_COMPARATOR);
616       // Verify there are no duplicate split keys
617       byte [] lastKey = null;
618       for(byte [] splitKey : splitKeys) {
619         if (Bytes.compareTo(splitKey, HConstants.EMPTY_BYTE_ARRAY) == 0) {
620           throw new IllegalArgumentException(
621               "Empty split key must not be passed in the split keys.");
622         }
623         if(lastKey != null && Bytes.equals(splitKey, lastKey)) {
624           throw new IllegalArgumentException("All split keys must be unique, " +
625             "found duplicate: " + Bytes.toStringBinary(splitKey) +
626             ", " + Bytes.toStringBinary(lastKey));
627         }
628         lastKey = splitKey;
629       }
630     }
631 
632     executeCallable(new MasterCallable<Void>(getConnection()) {
633       @Override
634       public Void call() throws ServiceException {
635         CreateTableRequest request = RequestConverter.buildCreateTableRequest(desc, splitKeys);
636         master.createTable(null, request);
637         return null;
638       }
639     });
640   }
641 
642   public void deleteTable(final String tableName) throws IOException {
643     deleteTable(TableName.valueOf(tableName));
644   }
645 
646   public void deleteTable(final byte[] tableName) throws IOException {
647     deleteTable(TableName.valueOf(tableName));
648   }
649 
650   /**
651    * Deletes a table.
652    * Synchronous operation.
653    *
654    * @param tableName name of table to delete
655    * @throws IOException if a remote or network exception occurs
656    */
657   public void deleteTable(final TableName tableName) throws IOException {
658     boolean tableExists = true;
659 
660     executeCallable(new MasterCallable<Void>(getConnection()) {
661       @Override
662       public Void call() throws ServiceException {
663         DeleteTableRequest req = RequestConverter.buildDeleteTableRequest(tableName);
664         master.deleteTable(null,req);
665         return null;
666       }
667     });
668 
669     int failures = 0;
670     // Wait until all regions deleted
671     for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
672       try {
673         HRegionLocation firstMetaServer = getFirstMetaServerForTable(tableName);
674         Scan scan = MetaReader.getScanForTableName(tableName);
675         scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
676         ScanRequest request = RequestConverter.buildScanRequest(
677           firstMetaServer.getRegionInfo().getRegionName(), scan, 1, true);
678         Result[] values = null;
679         // Get a batch at a time.
680         ClientService.BlockingInterface server = connection.getClient(firstMetaServer
681             .getServerName());
682         PayloadCarryingRpcController controller = new PayloadCarryingRpcController();
683         try {
684           controller.setPriority(tableName);
685           ScanResponse response = server.scan(controller, request);
686           values = ResponseConverter.getResults(controller.cellScanner(), response);
687         } catch (ServiceException se) {
688           throw ProtobufUtil.getRemoteException(se);
689         }
690 
691         // let us wait until hbase:meta table is updated and
692         // HMaster removes the table from its HTableDescriptors
693         if (values == null || values.length == 0) {
694           tableExists = false;
695           GetTableDescriptorsResponse htds;
696           MasterKeepAliveConnection master = connection.getKeepAliveMasterService();
697           try {
698             GetTableDescriptorsRequest req =
699               RequestConverter.buildGetTableDescriptorsRequest(tableName);
700             htds = master.getTableDescriptors(null, req);
701           } catch (ServiceException se) {
702             throw ProtobufUtil.getRemoteException(se);
703           } finally {
704             master.close();
705           }
706           tableExists = !htds.getTableSchemaList().isEmpty();
707           if (!tableExists) {
708             break;
709           }
710         }
711       } catch (IOException ex) {
712         failures++;
713         if(failures == numRetries - 1) {           // no more tries left
714           if (ex instanceof RemoteException) {
715             throw ((RemoteException) ex).unwrapRemoteException();
716           } else {
717             throw ex;
718           }
719         }
720       }
721       try {
722         Thread.sleep(getPauseTime(tries));
723       } catch (InterruptedException e) {
724         // continue
725       }
726     }
727 
728     if (tableExists) {
729       throw new IOException("Retries exhausted, it took too long to wait"+
730         " for the table " + tableName + " to be deleted.");
731     }
732     // Delete cached information to prevent clients from using old locations
733     this.connection.clearRegionCache(tableName);
734     LOG.info("Deleted " + tableName);
735   }
736 
737   /**
738    * Deletes tables matching the passed in pattern and wait on completion.
739    *
740    * Warning: Use this method carefully, there is no prompting and the effect is
741    * immediate. Consider using {@link #listTables(java.lang.String)} and
742    * {@link #deleteTable(byte[])}
743    *
744    * @param regex The regular expression to match table names against
745    * @return Table descriptors for tables that couldn't be deleted
746    * @throws IOException
747    * @see #deleteTables(java.util.regex.Pattern)
748    * @see #deleteTable(java.lang.String)
749    */
750   public HTableDescriptor[] deleteTables(String regex) throws IOException {
751     return deleteTables(Pattern.compile(regex));
752   }
753 
754   /**
755    * Delete tables matching the passed in pattern and wait on completion.
756    *
757    * Warning: Use this method carefully, there is no prompting and the effect is
758    * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
759    * {@link #deleteTable(byte[])}
760    *
761    * @param pattern The pattern to match table names against
762    * @return Table descriptors for tables that couldn't be deleted
763    * @throws IOException
764    */
765   public HTableDescriptor[] deleteTables(Pattern pattern) throws IOException {
766     List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
767     for (HTableDescriptor table : listTables(pattern)) {
768       try {
769         deleteTable(table.getTableName());
770       } catch (IOException ex) {
771         LOG.info("Failed to delete table " + table.getTableName(), ex);
772         failed.add(table);
773       }
774     }
775     return failed.toArray(new HTableDescriptor[failed.size()]);
776   }
777 
778 
779   /**
780    * Enable a table.  May timeout.  Use {@link #enableTableAsync(byte[])}
781    * and {@link #isTableEnabled(byte[])} instead.
782    * The table has to be in disabled state for it to be enabled.
783    * @param tableName name of the table
784    * @throws IOException if a remote or network exception occurs
785    * There could be couple types of IOException
786    * TableNotFoundException means the table doesn't exist.
787    * TableNotDisabledException means the table isn't in disabled state.
788    * @see #isTableEnabled(byte[])
789    * @see #disableTable(byte[])
790    * @see #enableTableAsync(byte[])
791    */
792   public void enableTable(final TableName tableName)
793   throws IOException {
794     enableTableAsync(tableName);
795 
796     // Wait until all regions are enabled
797     waitUntilTableIsEnabled(tableName);
798 
799     LOG.info("Enabled table " + tableName);
800   }
801 
802   public void enableTable(final byte[] tableName)
803   throws IOException {
804     enableTable(TableName.valueOf(tableName));
805   }
806 
807   public void enableTable(final String tableName)
808   throws IOException {
809     enableTable(TableName.valueOf(tableName));
810   }
811 
812   /**
813    * Wait for the table to be enabled and available
814    * If enabling the table exceeds the retry period, an exception is thrown.
815    * @param tableName name of the table
816    * @throws IOException if a remote or network exception occurs or
817    *    table is not enabled after the retries period.
818    */
819   private void waitUntilTableIsEnabled(final TableName tableName) throws IOException {
820     boolean enabled = false;
821     long start = EnvironmentEdgeManager.currentTimeMillis();
822     for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
823       try {
824         enabled = isTableEnabled(tableName);
825       } catch (TableNotFoundException tnfe) {
826         // wait for table to be created
827         enabled = false;
828       }
829       enabled = enabled && isTableAvailable(tableName);
830       if (enabled) {
831         break;
832       }
833       long sleep = getPauseTime(tries);
834       if (LOG.isDebugEnabled()) {
835         LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " +
836           "enabled in " + tableName);
837       }
838       try {
839         Thread.sleep(sleep);
840       } catch (InterruptedException e) {
841         // Do this conversion rather than let it out because do not want to
842         // change the method signature.
843         throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
844       }
845     }
846     if (!enabled) {
847       long msec = EnvironmentEdgeManager.currentTimeMillis() - start;
848       throw new IOException("Table '" + tableName +
849         "' not yet enabled, after " + msec + "ms.");
850     }
851   }
852 
853   /**
854    * Brings a table on-line (enables it).  Method returns immediately though
855    * enable of table may take some time to complete, especially if the table
856    * is large (All regions are opened as part of enabling process).  Check
857    * {@link #isTableEnabled(byte[])} to learn when table is fully online.  If
858    * table is taking too long to online, check server logs.
859    * @param tableName
860    * @throws IOException
861    * @since 0.90.0
862    */
863   public void enableTableAsync(final TableName tableName)
864   throws IOException {
865     TableName.isLegalFullyQualifiedTableName(tableName.getName());
866     executeCallable(new MasterCallable<Void>(getConnection()) {
867       @Override
868       public Void call() throws ServiceException {
869         LOG.info("Started enable of " + tableName);
870         EnableTableRequest req = RequestConverter.buildEnableTableRequest(tableName);
871         master.enableTable(null,req);
872         return null;
873       }
874     });
875   }
876 
877   public void enableTableAsync(final byte[] tableName)
878   throws IOException {
879     enableTable(TableName.valueOf(tableName));
880   }
881 
882   public void enableTableAsync(final String tableName)
883   throws IOException {
884     enableTableAsync(TableName.valueOf(tableName));
885   }
886 
887   /**
888    * Enable tables matching the passed in pattern and wait on completion.
889    *
890    * Warning: Use this method carefully, there is no prompting and the effect is
891    * immediate. Consider using {@link #listTables(java.lang.String)} and
892    * {@link #enableTable(byte[])}
893    *
894    * @param regex The regular expression to match table names against
895    * @throws IOException
896    * @see #enableTables(java.util.regex.Pattern)
897    * @see #enableTable(java.lang.String)
898    */
899   public HTableDescriptor[] enableTables(String regex) throws IOException {
900     return enableTables(Pattern.compile(regex));
901   }
902 
903   /**
904    * Enable tables matching the passed in pattern and wait on completion.
905    *
906    * Warning: Use this method carefully, there is no prompting and the effect is
907    * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
908    * {@link #enableTable(byte[])}
909    *
910    * @param pattern The pattern to match table names against
911    * @throws IOException
912    */
913   public HTableDescriptor[] enableTables(Pattern pattern) throws IOException {
914     List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
915     for (HTableDescriptor table : listTables(pattern)) {
916       if (isTableDisabled(table.getTableName())) {
917         try {
918           enableTable(table.getTableName());
919         } catch (IOException ex) {
920           LOG.info("Failed to enable table " + table.getTableName(), ex);
921           failed.add(table);
922         }
923       }
924     }
925     return failed.toArray(new HTableDescriptor[failed.size()]);
926   }
927 
928   /**
929    * Starts the disable of a table.  If it is being served, the master
930    * will tell the servers to stop serving it.  This method returns immediately.
931    * The disable of a table can take some time if the table is large (all
932    * regions are closed as part of table disable operation).
933    * Call {@link #isTableDisabled(byte[])} to check for when disable completes.
934    * If table is taking too long to online, check server logs.
935    * @param tableName name of table
936    * @throws IOException if a remote or network exception occurs
937    * @see #isTableDisabled(byte[])
938    * @see #isTableEnabled(byte[])
939    * @since 0.90.0
940    */
941   public void disableTableAsync(final TableName tableName) throws IOException {
942     TableName.isLegalFullyQualifiedTableName(tableName.getName());
943     executeCallable(new MasterCallable<Void>(getConnection()) {
944       @Override
945       public Void call() throws ServiceException {
946         LOG.info("Started disable of " + tableName);
947         DisableTableRequest req = RequestConverter.buildDisableTableRequest(tableName);
948         master.disableTable(null,req);
949         return null;
950       }
951     });
952   }
953 
954   public void disableTableAsync(final byte[] tableName) throws IOException {
955     disableTableAsync(TableName.valueOf(tableName));
956   }
957 
958   public void disableTableAsync(final String tableName) throws IOException {
959     disableTableAsync(TableName.valueOf(tableName));
960   }
961 
962   /**
963    * Disable table and wait on completion.  May timeout eventually.  Use
964    * {@link #disableTableAsync(byte[])} and {@link #isTableDisabled(String)}
965    * instead.
966    * The table has to be in enabled state for it to be disabled.
967    * @param tableName
968    * @throws IOException
969    * There could be couple types of IOException
970    * TableNotFoundException means the table doesn't exist.
971    * TableNotEnabledException means the table isn't in enabled state.
972    */
973   public void disableTable(final TableName tableName)
974   throws IOException {
975     disableTableAsync(tableName);
976     // Wait until table is disabled
977     boolean disabled = false;
978     for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
979       disabled = isTableDisabled(tableName);
980       if (disabled) {
981         break;
982       }
983       long sleep = getPauseTime(tries);
984       if (LOG.isDebugEnabled()) {
985         LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " +
986           "disabled in " + tableName);
987       }
988       try {
989         Thread.sleep(sleep);
990       } catch (InterruptedException e) {
991         // Do this conversion rather than let it out because do not want to
992         // change the method signature.
993         throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
994       }
995     }
996     if (!disabled) {
997       throw new RegionException("Retries exhausted, it took too long to wait"+
998         " for the table " + tableName + " to be disabled.");
999     }
1000     LOG.info("Disabled " + tableName);
1001   }
1002 
1003   public void disableTable(final byte[] tableName)
1004   throws IOException {
1005     disableTable(TableName.valueOf(tableName));
1006   }
1007 
1008   public void disableTable(final String tableName)
1009   throws IOException {
1010     disableTable(TableName.valueOf(tableName));
1011   }
1012 
1013   /**
1014    * Disable tables matching the passed in pattern and wait on completion.
1015    *
1016    * Warning: Use this method carefully, there is no prompting and the effect is
1017    * immediate. Consider using {@link #listTables(java.lang.String)} and
1018    * {@link #disableTable(byte[])}
1019    *
1020    * @param regex The regular expression to match table names against
1021    * @return Table descriptors for tables that couldn't be disabled
1022    * @throws IOException
1023    * @see #disableTables(java.util.regex.Pattern)
1024    * @see #disableTable(java.lang.String)
1025    */
1026   public HTableDescriptor[] disableTables(String regex) throws IOException {
1027     return disableTables(Pattern.compile(regex));
1028   }
1029 
1030   /**
1031    * Disable tables matching the passed in pattern and wait on completion.
1032    *
1033    * Warning: Use this method carefully, there is no prompting and the effect is
1034    * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
1035    * {@link #disableTable(byte[])}
1036    *
1037    * @param pattern The pattern to match table names against
1038    * @return Table descriptors for tables that couldn't be disabled
1039    * @throws IOException
1040    */
1041   public HTableDescriptor[] disableTables(Pattern pattern) throws IOException {
1042     List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
1043     for (HTableDescriptor table : listTables(pattern)) {
1044       if (isTableEnabled(table.getTableName())) {
1045         try {
1046           disableTable(table.getTableName());
1047         } catch (IOException ex) {
1048           LOG.info("Failed to disable table " + table.getTableName(), ex);
1049           failed.add(table);
1050         }
1051       }
1052     }
1053     return failed.toArray(new HTableDescriptor[failed.size()]);
1054   }
1055 
1056   /*
1057    * Checks whether table exists. If not, throws TableNotFoundException
1058    * @param tableName
1059    */
1060   private void checkTableExistence(TableName tableName) throws IOException {
1061     if (!tableExists(tableName)) {
1062       throw new TableNotFoundException(tableName);
1063     }
1064   }
1065 
1066   /**
1067    * @param tableName name of table to check
1068    * @return true if table is on-line
1069    * @throws IOException if a remote or network exception occurs
1070    */
1071   public boolean isTableEnabled(TableName tableName) throws IOException {
1072     checkTableExistence(tableName);
1073     return connection.isTableEnabled(tableName);
1074   }
1075 
1076   public boolean isTableEnabled(byte[] tableName) throws IOException {
1077     return isTableEnabled(TableName.valueOf(tableName));
1078   }
1079 
1080   public boolean isTableEnabled(String tableName) throws IOException {
1081     return isTableEnabled(TableName.valueOf(tableName));
1082   }
1083 
1084 
1085 
1086   /**
1087    * @param tableName name of table to check
1088    * @return true if table is off-line
1089    * @throws IOException if a remote or network exception occurs
1090    */
1091   public boolean isTableDisabled(TableName tableName) throws IOException {
1092     checkTableExistence(tableName);
1093     return connection.isTableDisabled(tableName);
1094   }
1095 
1096   public boolean isTableDisabled(byte[] tableName) throws IOException {
1097     return isTableDisabled(TableName.valueOf(tableName));
1098   }
1099 
1100   public boolean isTableDisabled(String tableName) throws IOException {
1101     return isTableDisabled(TableName.valueOf(tableName));
1102   }
1103 
1104   /**
1105    * @param tableName name of table to check
1106    * @return true if all regions of the table are available
1107    * @throws IOException if a remote or network exception occurs
1108    */
1109   public boolean isTableAvailable(TableName tableName) throws IOException {
1110     return connection.isTableAvailable(tableName);
1111   }
1112 
1113   public boolean isTableAvailable(byte[] tableName) throws IOException {
1114     return isTableAvailable(TableName.valueOf(tableName));
1115   }
1116 
1117   public boolean isTableAvailable(String tableName) throws IOException {
1118     return isTableAvailable(TableName.valueOf(tableName));
1119   }
1120 
1121   /**
1122    * Use this api to check if the table has been created with the specified number of
1123    * splitkeys which was used while creating the given table.
1124    * Note : If this api is used after a table's region gets splitted, the api may return
1125    * false.
1126    * @param tableName
1127    *          name of table to check
1128    * @param splitKeys
1129    *          keys to check if the table has been created with all split keys
1130    * @throws IOException
1131    *           if a remote or network excpetion occurs
1132    */
1133   public boolean isTableAvailable(TableName tableName,
1134                                   byte[][] splitKeys) throws IOException {
1135     return connection.isTableAvailable(tableName, splitKeys);
1136   }
1137 
1138   public boolean isTableAvailable(byte[] tableName,
1139                                   byte[][] splitKeys) throws IOException {
1140     return isTableAvailable(TableName.valueOf(tableName), splitKeys);
1141   }
1142 
1143   public boolean isTableAvailable(String tableName,
1144                                   byte[][] splitKeys) throws IOException {
1145     return isTableAvailable(TableName.valueOf(tableName), splitKeys);
1146   }
1147 
1148   /**
1149    * Get the status of alter command - indicates how many regions have received
1150    * the updated schema Asynchronous operation.
1151    *
1152    * @param tableName TableName instance
1153    * @return Pair indicating the number of regions updated Pair.getFirst() is the
1154    *         regions that are yet to be updated Pair.getSecond() is the total number
1155    *         of regions of the table
1156    * @throws IOException
1157    *           if a remote or network exception occurs
1158    */
1159   public Pair<Integer, Integer> getAlterStatus(final TableName tableName)
1160   throws IOException {
1161     return executeCallable(new MasterCallable<Pair<Integer, Integer>>(getConnection()) {
1162       @Override
1163       public Pair<Integer, Integer> call() throws ServiceException {
1164         GetSchemaAlterStatusRequest req = RequestConverter
1165             .buildGetSchemaAlterStatusRequest(tableName);
1166         GetSchemaAlterStatusResponse ret = master.getSchemaAlterStatus(null, req);
1167         Pair<Integer, Integer> pair = new Pair<Integer, Integer>(Integer.valueOf(ret
1168             .getYetToUpdateRegions()), Integer.valueOf(ret.getTotalRegions()));
1169         return pair;
1170       }
1171     });
1172   }
1173 
1174   /**
1175    * Get the status of alter command - indicates how many regions have received
1176    * the updated schema Asynchronous operation.
1177    *
1178    * @param tableName
1179    *          name of the table to get the status of
1180    * @return Pair indicating the number of regions updated Pair.getFirst() is the
1181    *         regions that are yet to be updated Pair.getSecond() is the total number
1182    *         of regions of the table
1183    * @throws IOException
1184    *           if a remote or network exception occurs
1185    */
1186   public Pair<Integer, Integer> getAlterStatus(final byte[] tableName)
1187    throws IOException {
1188     return getAlterStatus(TableName.valueOf(tableName));
1189   }
1190 
1191   /**
1192    * Add a column to an existing table.
1193    * Asynchronous operation.
1194    *
1195    * @param tableName name of the table to add column to
1196    * @param column column descriptor of column to be added
1197    * @throws IOException if a remote or network exception occurs
1198    */
1199   public void addColumn(final byte[] tableName, HColumnDescriptor column)
1200   throws IOException {
1201     addColumn(TableName.valueOf(tableName), column);
1202   }
1203 
1204 
1205   /**
1206    * Add a column to an existing table.
1207    * Asynchronous operation.
1208    *
1209    * @param tableName name of the table to add column to
1210    * @param column column descriptor of column to be added
1211    * @throws IOException if a remote or network exception occurs
1212    */
1213   public void addColumn(final String tableName, HColumnDescriptor column)
1214   throws IOException {
1215     addColumn(TableName.valueOf(tableName), column);
1216   }
1217 
1218   /**
1219    * Add a column to an existing table.
1220    * Asynchronous operation.
1221    *
1222    * @param tableName name of the table to add column to
1223    * @param column column descriptor of column to be added
1224    * @throws IOException if a remote or network exception occurs
1225    */
1226   public void addColumn(final TableName tableName, final HColumnDescriptor column)
1227   throws IOException {
1228     executeCallable(new MasterCallable<Void>(getConnection()) {
1229       @Override
1230       public Void call() throws ServiceException {
1231         AddColumnRequest req = RequestConverter.buildAddColumnRequest(tableName, column);
1232         master.addColumn(null,req);
1233         return null;
1234       }
1235     });
1236   }
1237 
1238   /**
1239    * Delete a column from a table.
1240    * Asynchronous operation.
1241    *
1242    * @param tableName name of table
1243    * @param columnName name of column to be deleted
1244    * @throws IOException if a remote or network exception occurs
1245    */
1246   public void deleteColumn(final byte[] tableName, final String columnName)
1247   throws IOException {
1248     deleteColumn(TableName.valueOf(tableName), Bytes.toBytes(columnName));
1249   }
1250 
1251   /**
1252    * Delete a column from a table.
1253    * Asynchronous operation.
1254    *
1255    * @param tableName name of table
1256    * @param columnName name of column to be deleted
1257    * @throws IOException if a remote or network exception occurs
1258    */
1259   public void deleteColumn(final String tableName, final String columnName)
1260   throws IOException {
1261     deleteColumn(TableName.valueOf(tableName), Bytes.toBytes(columnName));
1262   }
1263 
1264   /**
1265    * Delete a column from a table.
1266    * Asynchronous operation.
1267    *
1268    * @param tableName name of table
1269    * @param columnName name of column to be deleted
1270    * @throws IOException if a remote or network exception occurs
1271    */
1272   public void deleteColumn(final TableName tableName, final byte [] columnName)
1273   throws IOException {
1274     executeCallable(new MasterCallable<Void>(getConnection()) {
1275       @Override
1276       public Void call() throws ServiceException {
1277         DeleteColumnRequest req = RequestConverter.buildDeleteColumnRequest(tableName, columnName);
1278         master.deleteColumn(null,req);
1279         return null;
1280       }
1281     });
1282   }
1283 
1284   /**
1285    * Modify an existing column family on a table.
1286    * Asynchronous operation.
1287    *
1288    * @param tableName name of table
1289    * @param descriptor new column descriptor to use
1290    * @throws IOException if a remote or network exception occurs
1291    */
1292   public void modifyColumn(final String tableName, HColumnDescriptor descriptor)
1293   throws IOException {
1294     modifyColumn(TableName.valueOf(tableName), descriptor);
1295   }
1296 
1297   /**
1298    * Modify an existing column family on a table.
1299    * Asynchronous operation.
1300    *
1301    * @param tableName name of table
1302    * @param descriptor new column descriptor to use
1303    * @throws IOException if a remote or network exception occurs
1304    */
1305   public void modifyColumn(final byte[] tableName, HColumnDescriptor descriptor)
1306   throws IOException {
1307     modifyColumn(TableName.valueOf(tableName), descriptor);
1308   }
1309 
1310 
1311 
1312   /**
1313    * Modify an existing column family on a table.
1314    * Asynchronous operation.
1315    *
1316    * @param tableName name of table
1317    * @param descriptor new column descriptor to use
1318    * @throws IOException if a remote or network exception occurs
1319    */
1320   public void modifyColumn(final TableName tableName, final HColumnDescriptor descriptor)
1321   throws IOException {
1322     executeCallable(new MasterCallable<Void>(getConnection()) {
1323       @Override
1324       public Void call() throws ServiceException {
1325         ModifyColumnRequest req = RequestConverter.buildModifyColumnRequest(tableName, descriptor);
1326         master.modifyColumn(null,req);
1327         return null;
1328       }
1329     });
1330   }
1331 
1332   /**
1333    * Close a region. For expert-admins.  Runs close on the regionserver.  The
1334    * master will not be informed of the close.
1335    * @param regionname region name to close
1336    * @param serverName If supplied, we'll use this location rather than
1337    * the one currently in <code>hbase:meta</code>
1338    * @throws IOException if a remote or network exception occurs
1339    */
1340   public void closeRegion(final String regionname, final String serverName)
1341   throws IOException {
1342     closeRegion(Bytes.toBytes(regionname), serverName);
1343   }
1344 
1345   /**
1346    * Close a region.  For expert-admins  Runs close on the regionserver.  The
1347    * master will not be informed of the close.
1348    * @param regionname region name to close
1349    * @param serverName The servername of the regionserver.  If passed null we
1350    * will use servername found in the hbase:meta table. A server name
1351    * is made of host, port and startcode.  Here is an example:
1352    * <code> host187.example.com,60020,1289493121758</code>
1353    * @throws IOException if a remote or network exception occurs
1354    */
1355   public void closeRegion(final byte [] regionname, final String serverName)
1356   throws IOException {
1357     CatalogTracker ct = getCatalogTracker();
1358     try {
1359       if (serverName != null) {
1360         Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, regionname);
1361         if (pair == null || pair.getFirst() == null) {
1362           throw new UnknownRegionException(Bytes.toStringBinary(regionname));
1363         } else {
1364           closeRegion(ServerName.valueOf(serverName), pair.getFirst());
1365         }
1366       } else {
1367         Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, regionname);
1368         if (pair == null) {
1369           throw new UnknownRegionException(Bytes.toStringBinary(regionname));
1370         } else if (pair.getSecond() == null) {
1371           throw new NoServerForRegionException(Bytes.toStringBinary(regionname));
1372         } else {
1373           closeRegion(pair.getSecond(), pair.getFirst());
1374         }
1375       }
1376     } finally {
1377       cleanupCatalogTracker(ct);
1378     }
1379   }
1380 
1381   /**
1382    * For expert-admins. Runs close on the regionserver. Closes a region based on
1383    * the encoded region name. The region server name is mandatory. If the
1384    * servername is provided then based on the online regions in the specified
1385    * regionserver the specified region will be closed. The master will not be
1386    * informed of the close. Note that the regionname is the encoded regionname.
1387    *
1388    * @param encodedRegionName
1389    *          The encoded region name; i.e. the hash that makes up the region
1390    *          name suffix: e.g. if regionname is
1391    *          <code>TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396.</code>
1392    *          , then the encoded region name is:
1393    *          <code>527db22f95c8a9e0116f0cc13c680396</code>.
1394    * @param serverName
1395    *          The servername of the regionserver. A server name is made of host,
1396    *          port and startcode. This is mandatory. Here is an example:
1397    *          <code> host187.example.com,60020,1289493121758</code>
1398    * @return true if the region was closed, false if not.
1399    * @throws IOException
1400    *           if a remote or network exception occurs
1401    */
1402   public boolean closeRegionWithEncodedRegionName(final String encodedRegionName,
1403       final String serverName) throws IOException {
1404     if (null == serverName || ("").equals(serverName.trim())) {
1405       throw new IllegalArgumentException(
1406           "The servername cannot be null or empty.");
1407     }
1408     ServerName sn = ServerName.valueOf(serverName);
1409     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1410     // Close the region without updating zk state.
1411     CloseRegionRequest request =
1412       RequestConverter.buildCloseRegionRequest(sn, encodedRegionName, false);
1413     try {
1414       CloseRegionResponse response = admin.closeRegion(null, request);
1415       boolean isRegionClosed = response.getClosed();
1416       if (false == isRegionClosed) {
1417         LOG.error("Not able to close the region " + encodedRegionName + ".");
1418       }
1419       return isRegionClosed;
1420     } catch (ServiceException se) {
1421       throw ProtobufUtil.getRemoteException(se);
1422     }
1423   }
1424 
1425   /**
1426    * Close a region.  For expert-admins  Runs close on the regionserver.  The
1427    * master will not be informed of the close.
1428    * @param sn
1429    * @param hri
1430    * @throws IOException
1431    */
1432   public void closeRegion(final ServerName sn, final HRegionInfo hri)
1433   throws IOException {
1434     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1435     // Close the region without updating zk state.
1436     ProtobufUtil.closeRegion(admin, sn, hri.getRegionName(), false);
1437   }
1438 
1439   /**
1440    * Get all the online regions on a region server.
1441    */
1442   public List<HRegionInfo> getOnlineRegions(
1443       final ServerName sn) throws IOException {
1444     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1445     return ProtobufUtil.getOnlineRegions(admin);
1446   }
1447 
1448   /**
1449    * Flush a table or an individual region.
1450    * Synchronous operation.
1451    *
1452    * @param tableNameOrRegionName table or region to flush
1453    * @throws IOException if a remote or network exception occurs
1454    * @throws InterruptedException
1455    */
1456   public void flush(final String tableNameOrRegionName)
1457   throws IOException, InterruptedException {
1458     flush(Bytes.toBytes(tableNameOrRegionName));
1459   }
1460 
1461   /**
1462    * Flush a table or an individual region.
1463    * Synchronous operation.
1464    *
1465    * @param tableNameOrRegionName table or region to flush
1466    * @throws IOException if a remote or network exception occurs
1467    * @throws InterruptedException
1468    */
1469   public void flush(final byte[] tableNameOrRegionName)
1470   throws IOException, InterruptedException {
1471     CatalogTracker ct = getCatalogTracker();
1472     try {
1473       Pair<HRegionInfo, ServerName> regionServerPair
1474         = getRegion(tableNameOrRegionName, ct);
1475       if (regionServerPair != null) {
1476         if (regionServerPair.getSecond() == null) {
1477           throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
1478         } else {
1479           flush(regionServerPair.getSecond(), regionServerPair.getFirst());
1480         }
1481       } else {
1482         final TableName tableName = checkTableExists(
1483             TableName.valueOf(tableNameOrRegionName), ct);
1484         List<Pair<HRegionInfo, ServerName>> pairs =
1485           MetaReader.getTableRegionsAndLocations(ct,
1486               tableName);
1487         for (Pair<HRegionInfo, ServerName> pair: pairs) {
1488           if (pair.getFirst().isOffline()) continue;
1489           if (pair.getSecond() == null) continue;
1490           try {
1491             flush(pair.getSecond(), pair.getFirst());
1492           } catch (NotServingRegionException e) {
1493             if (LOG.isDebugEnabled()) {
1494               LOG.debug("Trying to flush " + pair.getFirst() + ": " +
1495                 StringUtils.stringifyException(e));
1496             }
1497           }
1498         }
1499       }
1500     } finally {
1501       cleanupCatalogTracker(ct);
1502     }
1503   }
1504 
1505   private void flush(final ServerName sn, final HRegionInfo hri)
1506   throws IOException {
1507     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1508     FlushRegionRequest request =
1509       RequestConverter.buildFlushRegionRequest(hri.getRegionName());
1510     try {
1511       admin.flushRegion(null, request);
1512     } catch (ServiceException se) {
1513       throw ProtobufUtil.getRemoteException(se);
1514     }
1515   }
1516 
1517   /**
1518    * Compact a table or an individual region.
1519    * Asynchronous operation.
1520    *
1521    * @param tableNameOrRegionName table or region to compact
1522    * @throws IOException if a remote or network exception occurs
1523    * @throws InterruptedException
1524    */
1525   public void compact(final String tableNameOrRegionName)
1526   throws IOException, InterruptedException {
1527     compact(Bytes.toBytes(tableNameOrRegionName));
1528   }
1529 
1530   /**
1531    * Compact a table or an individual region.
1532    * Asynchronous operation.
1533    *
1534    * @param tableNameOrRegionName table or region to compact
1535    * @throws IOException if a remote or network exception occurs
1536    * @throws InterruptedException
1537    */
1538   public void compact(final byte[] tableNameOrRegionName)
1539   throws IOException, InterruptedException {
1540     compact(tableNameOrRegionName, null, false);
1541   }
1542 
1543   /**
1544    * Compact a column family within a table or region.
1545    * Asynchronous operation.
1546    *
1547    * @param tableOrRegionName table or region to compact
1548    * @param columnFamily column family within a table or region
1549    * @throws IOException if a remote or network exception occurs
1550    * @throws InterruptedException
1551    */
1552   public void compact(String tableOrRegionName, String columnFamily)
1553     throws IOException,  InterruptedException {
1554     compact(Bytes.toBytes(tableOrRegionName), Bytes.toBytes(columnFamily));
1555   }
1556 
1557   /**
1558    * Compact a column family within a table or region.
1559    * Asynchronous operation.
1560    *
1561    * @param tableNameOrRegionName table or region to compact
1562    * @param columnFamily column family within a table or region
1563    * @throws IOException if a remote or network exception occurs
1564    * @throws InterruptedException
1565    */
1566   public void compact(final byte[] tableNameOrRegionName, final byte[] columnFamily)
1567   throws IOException, InterruptedException {
1568     compact(tableNameOrRegionName, columnFamily, false);
1569   }
1570 
1571   /**
1572    * Major compact a table or an individual region.
1573    * Asynchronous operation.
1574    *
1575    * @param tableNameOrRegionName table or region to major compact
1576    * @throws IOException if a remote or network exception occurs
1577    * @throws InterruptedException
1578    */
1579   public void majorCompact(final String tableNameOrRegionName)
1580   throws IOException, InterruptedException {
1581     majorCompact(Bytes.toBytes(tableNameOrRegionName));
1582   }
1583 
1584   /**
1585    * Major compact a table or an individual region.
1586    * Asynchronous operation.
1587    *
1588    * @param tableNameOrRegionName table or region to major compact
1589    * @throws IOException if a remote or network exception occurs
1590    * @throws InterruptedException
1591    */
1592   public void majorCompact(final byte[] tableNameOrRegionName)
1593   throws IOException, InterruptedException {
1594     compact(tableNameOrRegionName, null, true);
1595   }
1596 
1597   /**
1598    * Major compact a column family within a table or region.
1599    * Asynchronous operation.
1600    *
1601    * @param tableNameOrRegionName table or region to major compact
1602    * @param columnFamily column family within a table or region
1603    * @throws IOException if a remote or network exception occurs
1604    * @throws InterruptedException
1605    */
1606   public void majorCompact(final String tableNameOrRegionName,
1607     final String columnFamily) throws IOException, InterruptedException {
1608     majorCompact(Bytes.toBytes(tableNameOrRegionName),
1609       Bytes.toBytes(columnFamily));
1610   }
1611 
1612   /**
1613    * Major compact a column family within a table or region.
1614    * Asynchronous operation.
1615    *
1616    * @param tableNameOrRegionName table or region to major compact
1617    * @param columnFamily column family within a table or region
1618    * @throws IOException if a remote or network exception occurs
1619    * @throws InterruptedException
1620    */
1621   public void majorCompact(final byte[] tableNameOrRegionName,
1622     final byte[] columnFamily) throws IOException, InterruptedException {
1623     compact(tableNameOrRegionName, columnFamily, true);
1624   }
1625 
1626   /**
1627    * Compact a table or an individual region.
1628    * Asynchronous operation.
1629    *
1630    * @param tableNameOrRegionName table or region to compact
1631    * @param columnFamily column family within a table or region
1632    * @param major True if we are to do a major compaction.
1633    * @throws IOException if a remote or network exception occurs
1634    * @throws InterruptedException
1635    */
1636   private void compact(final byte[] tableNameOrRegionName,
1637     final byte[] columnFamily,final boolean major)
1638   throws IOException, InterruptedException {
1639     CatalogTracker ct = getCatalogTracker();
1640     try {
1641       Pair<HRegionInfo, ServerName> regionServerPair
1642         = getRegion(tableNameOrRegionName, ct);
1643       if (regionServerPair != null) {
1644         if (regionServerPair.getSecond() == null) {
1645           throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
1646         } else {
1647           compact(regionServerPair.getSecond(), regionServerPair.getFirst(), major, columnFamily);
1648         }
1649       } else {
1650         final TableName tableName =
1651             checkTableExists(TableName.valueOf(tableNameOrRegionName), ct);
1652         List<Pair<HRegionInfo, ServerName>> pairs =
1653           MetaReader.getTableRegionsAndLocations(ct,
1654               tableName);
1655         for (Pair<HRegionInfo, ServerName> pair: pairs) {
1656           if (pair.getFirst().isOffline()) continue;
1657           if (pair.getSecond() == null) continue;
1658           try {
1659             compact(pair.getSecond(), pair.getFirst(), major, columnFamily);
1660           } catch (NotServingRegionException e) {
1661             if (LOG.isDebugEnabled()) {
1662               LOG.debug("Trying to" + (major ? " major" : "") + " compact " +
1663                 pair.getFirst() + ": " +
1664                 StringUtils.stringifyException(e));
1665             }
1666           }
1667         }
1668       }
1669     } finally {
1670       cleanupCatalogTracker(ct);
1671     }
1672   }
1673 
1674   private void compact(final ServerName sn, final HRegionInfo hri,
1675       final boolean major, final byte [] family)
1676   throws IOException {
1677     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1678     CompactRegionRequest request =
1679       RequestConverter.buildCompactRegionRequest(hri.getRegionName(), major, family);
1680     try {
1681       admin.compactRegion(null, request);
1682     } catch (ServiceException se) {
1683       throw ProtobufUtil.getRemoteException(se);
1684     }
1685   }
1686 
1687   /**
1688    * Compact all regions on the region server
1689    * @param regionserver the region server name
1690    * @param major if it's major compaction
1691    * @throws IOException
1692    * @throws InterruptedException
1693    */
1694   public void compactRegionServer(final ServerName sn, boolean major)
1695       throws IOException, InterruptedException {
1696     for (HRegionInfo region : getOnlineRegions(sn)) {
1697       compact(sn, region, major, null);
1698     }
1699   }
1700 
1701   /**
1702    * Move the region <code>r</code> to <code>dest</code>.
1703    * @param encodedRegionName The encoded region name; i.e. the hash that makes
1704    * up the region name suffix: e.g. if regionname is
1705    * <code>TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396.</code>,
1706    * then the encoded region name is: <code>527db22f95c8a9e0116f0cc13c680396</code>.
1707    * @param destServerName The servername of the destination regionserver.  If
1708    * passed the empty byte array we'll assign to a random server.  A server name
1709    * is made of host, port and startcode.  Here is an example:
1710    * <code> host187.example.com,60020,1289493121758</code>
1711    * @throws UnknownRegionException Thrown if we can't find a region named
1712    * <code>encodedRegionName</code>
1713    * @throws ZooKeeperConnectionException
1714    * @throws MasterNotRunningException
1715    */
1716   public void move(final byte [] encodedRegionName, final byte [] destServerName)
1717   throws HBaseIOException, MasterNotRunningException, ZooKeeperConnectionException {
1718     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1719     try {
1720       MoveRegionRequest request =
1721         RequestConverter.buildMoveRegionRequest(encodedRegionName, destServerName);
1722       stub.moveRegion(null,request);
1723     } catch (ServiceException se) {
1724       IOException ioe = ProtobufUtil.getRemoteException(se);
1725       if (ioe instanceof HBaseIOException) {
1726         throw (HBaseIOException)ioe;
1727       }
1728       LOG.error("Unexpected exception: " + se + " from calling HMaster.moveRegion");
1729     } catch (DeserializationException de) {
1730       LOG.error("Could not parse destination server name: " + de);
1731     } finally {
1732       stub.close();
1733     }
1734   }
1735 
1736   /**
1737    * @param regionName
1738    *          Region name to assign.
1739    * @throws MasterNotRunningException
1740    * @throws ZooKeeperConnectionException
1741    * @throws IOException
1742    */
1743   public void assign(final byte[] regionName) throws MasterNotRunningException,
1744       ZooKeeperConnectionException, IOException {
1745     final byte[] toBeAssigned = getRegionName(regionName);
1746     executeCallable(new MasterCallable<Void>(getConnection()) {
1747       @Override
1748       public Void call() throws ServiceException {
1749         AssignRegionRequest request =
1750           RequestConverter.buildAssignRegionRequest(toBeAssigned);
1751         master.assignRegion(null,request);
1752         return null;
1753       }
1754     });
1755   }
1756 
1757   /**
1758    * Unassign a region from current hosting regionserver.  Region will then be
1759    * assigned to a regionserver chosen at random.  Region could be reassigned
1760    * back to the same server.  Use {@link #move(byte[], byte[])} if you want
1761    * to control the region movement.
1762    * @param regionName Region to unassign. Will clear any existing RegionPlan
1763    * if one found.
1764    * @param force If true, force unassign (Will remove region from
1765    * regions-in-transition too if present. If results in double assignment
1766    * use hbck -fix to resolve. To be used by experts).
1767    * @throws MasterNotRunningException
1768    * @throws ZooKeeperConnectionException
1769    * @throws IOException
1770    */
1771   public void unassign(final byte [] regionName, final boolean force)
1772   throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
1773     final byte[] toBeUnassigned = getRegionName(regionName);
1774     executeCallable(new MasterCallable<Void>(getConnection()) {
1775       @Override
1776       public Void call() throws ServiceException {
1777         UnassignRegionRequest request =
1778           RequestConverter.buildUnassignRegionRequest(toBeUnassigned, force);
1779         master.unassignRegion(null,request);
1780         return null;
1781       }
1782     });
1783   }
1784 
1785   /**
1786    * Offline specified region from master's in-memory state. It will not attempt to reassign the
1787    * region as in unassign. This API can be used when a region not served by any region server and
1788    * still online as per Master's in memory state. If this API is incorrectly used on active region
1789    * then master will loose track of that region.
1790    *
1791    * This is a special method that should be used by experts or hbck.
1792    *
1793    * @param regionName
1794    *          Region to offline.
1795    * @throws IOException
1796    */
1797   public void offline(final byte [] regionName)
1798   throws IOException {
1799     MasterKeepAliveConnection master = connection.getKeepAliveMasterService();
1800     try {
1801       master.offlineRegion(null,RequestConverter.buildOfflineRegionRequest(regionName));
1802     } catch (ServiceException se) {
1803       throw ProtobufUtil.getRemoteException(se);
1804     } finally {
1805       master.close();
1806     }
1807   }
1808 
1809   /**
1810    * Turn the load balancer on or off.
1811    * @param on If true, enable balancer. If false, disable balancer.
1812    * @param synchronous If true, it waits until current balance() call, if outstanding, to return.
1813    * @return Previous balancer value
1814    */
1815   public boolean setBalancerRunning(final boolean on, final boolean synchronous)
1816   throws MasterNotRunningException, ZooKeeperConnectionException {
1817     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1818     try {
1819       SetBalancerRunningRequest req =
1820         RequestConverter.buildSetBalancerRunningRequest(on, synchronous);
1821       return stub.setBalancerRunning(null, req).getPrevBalanceValue();
1822     } catch (ServiceException se) {
1823       IOException ioe = ProtobufUtil.getRemoteException(se);
1824       if (ioe instanceof MasterNotRunningException) {
1825         throw (MasterNotRunningException)ioe;
1826       }
1827       if (ioe instanceof ZooKeeperConnectionException) {
1828         throw (ZooKeeperConnectionException)ioe;
1829       }
1830 
1831       // Throwing MasterNotRunningException even though not really valid in order to not
1832       // break interface by adding additional exception type.
1833       throw new MasterNotRunningException("Unexpected exception when calling balanceSwitch",se);
1834     } finally {
1835       stub.close();
1836     }
1837   }
1838 
1839   /**
1840    * Invoke the balancer.  Will run the balancer and if regions to move, it will
1841    * go ahead and do the reassignments.  Can NOT run for various reasons.  Check
1842    * logs.
1843    * @return True if balancer ran, false otherwise.
1844    */
1845   public boolean balancer()
1846   throws MasterNotRunningException, ZooKeeperConnectionException, ServiceException {
1847     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1848     try {
1849       return stub.balance(null,RequestConverter.buildBalanceRequest()).getBalancerRan();
1850     } finally {
1851       stub.close();
1852     }
1853   }
1854 
1855   /**
1856    * Enable/Disable the catalog janitor
1857    * @param enable if true enables the catalog janitor
1858    * @return the previous state
1859    * @throws ServiceException
1860    * @throws MasterNotRunningException
1861    */
1862   public boolean enableCatalogJanitor(boolean enable)
1863       throws ServiceException, MasterNotRunningException {
1864     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1865     try {
1866       return stub.enableCatalogJanitor(null,
1867           RequestConverter.buildEnableCatalogJanitorRequest(enable)).getPrevValue();
1868     } finally {
1869       stub.close();
1870     }
1871   }
1872 
1873   /**
1874    * Ask for a scan of the catalog table
1875    * @return the number of entries cleaned
1876    * @throws ServiceException
1877    * @throws MasterNotRunningException
1878    */
1879   public int runCatalogScan() throws ServiceException, MasterNotRunningException {
1880     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1881     try {
1882       return stub.runCatalogScan(null,
1883           RequestConverter.buildCatalogScanRequest()).getScanResult();
1884     } finally {
1885       stub.close();
1886     }
1887   }
1888 
1889   /**
1890    * Query on the catalog janitor state (Enabled/Disabled?)
1891    * @throws ServiceException
1892    * @throws org.apache.hadoop.hbase.MasterNotRunningException
1893    */
1894   public boolean isCatalogJanitorEnabled() throws ServiceException, MasterNotRunningException {
1895     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1896     try {
1897       return stub.isCatalogJanitorEnabled(null,
1898           RequestConverter.buildIsCatalogJanitorEnabledRequest()).getValue();
1899     } finally {
1900       stub.close();
1901     }
1902   }
1903 
1904   /**
1905    * Merge two regions. Asynchronous operation.
1906    * @param encodedNameOfRegionA encoded name of region a
1907    * @param encodedNameOfRegionB encoded name of region b
1908    * @param forcible true if do a compulsory merge, otherwise we will only merge
1909    *          two adjacent regions
1910    * @throws IOException
1911    */
1912   public void mergeRegions(final byte[] encodedNameOfRegionA,
1913       final byte[] encodedNameOfRegionB, final boolean forcible)
1914       throws IOException {
1915     MasterKeepAliveConnection master = connection
1916         .getKeepAliveMasterService();
1917     try {
1918       DispatchMergingRegionsRequest request = RequestConverter
1919           .buildDispatchMergingRegionsRequest(encodedNameOfRegionA,
1920               encodedNameOfRegionB, forcible);
1921       master.dispatchMergingRegions(null, request);
1922     } catch (ServiceException se) {
1923       IOException ioe = ProtobufUtil.getRemoteException(se);
1924       if (ioe instanceof UnknownRegionException) {
1925         throw (UnknownRegionException) ioe;
1926       }
1927       if (ioe instanceof MergeRegionException) {
1928         throw (MergeRegionException) ioe;
1929       }
1930       LOG.error("Unexpected exception: " + se
1931           + " from calling HMaster.dispatchMergingRegions");
1932     } catch (DeserializationException de) {
1933       LOG.error("Could not parse destination server name: " + de);
1934     } finally {
1935       master.close();
1936     }
1937   }
1938 
1939   /**
1940    * Split a table or an individual region.
1941    * Asynchronous operation.
1942    *
1943    * @param tableNameOrRegionName table or region to split
1944    * @throws IOException if a remote or network exception occurs
1945    * @throws InterruptedException
1946    */
1947   public void split(final String tableNameOrRegionName)
1948   throws IOException, InterruptedException {
1949     split(Bytes.toBytes(tableNameOrRegionName));
1950   }
1951 
1952   /**
1953    * Split a table or an individual region.  Implicitly finds an optimal split
1954    * point.  Asynchronous operation.
1955    *
1956    * @param tableNameOrRegionName table to region to split
1957    * @throws IOException if a remote or network exception occurs
1958    * @throws InterruptedException
1959    */
1960   public void split(final byte[] tableNameOrRegionName)
1961   throws IOException, InterruptedException {
1962     split(tableNameOrRegionName, null);
1963   }
1964 
1965   public void split(final String tableNameOrRegionName,
1966     final String splitPoint) throws IOException, InterruptedException {
1967     split(Bytes.toBytes(tableNameOrRegionName), Bytes.toBytes(splitPoint));
1968   }
1969 
1970   /**
1971    * Split a table or an individual region.
1972    * Asynchronous operation.
1973    *
1974    * @param tableNameOrRegionName table to region to split
1975    * @param splitPoint the explicit position to split on
1976    * @throws IOException if a remote or network exception occurs
1977    * @throws InterruptedException interrupt exception occurred
1978    */
1979   public void split(final byte[] tableNameOrRegionName,
1980       final byte [] splitPoint) throws IOException, InterruptedException {
1981     CatalogTracker ct = getCatalogTracker();
1982     try {
1983       Pair<HRegionInfo, ServerName> regionServerPair
1984         = getRegion(tableNameOrRegionName, ct);
1985       if (regionServerPair != null) {
1986         if (regionServerPair.getSecond() == null) {
1987             throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
1988         } else {
1989           split(regionServerPair.getSecond(), regionServerPair.getFirst(), splitPoint);
1990         }
1991       } else {
1992         final TableName tableName =
1993             checkTableExists(TableName.valueOf(tableNameOrRegionName), ct);
1994         List<Pair<HRegionInfo, ServerName>> pairs =
1995           MetaReader.getTableRegionsAndLocations(ct,
1996               tableName);
1997         for (Pair<HRegionInfo, ServerName> pair: pairs) {
1998           // May not be a server for a particular row
1999           if (pair.getSecond() == null) continue;
2000           HRegionInfo r = pair.getFirst();
2001           // check for parents
2002           if (r.isSplitParent()) continue;
2003           // if a split point given, only split that particular region
2004           if (splitPoint != null && !r.containsRow(splitPoint)) continue;
2005           // call out to region server to do split now
2006           split(pair.getSecond(), pair.getFirst(), splitPoint);
2007         }
2008       }
2009     } finally {
2010       cleanupCatalogTracker(ct);
2011     }
2012   }
2013 
2014   private void split(final ServerName sn, final HRegionInfo hri,
2015       byte[] splitPoint) throws IOException {
2016     if (hri.getStartKey() != null && splitPoint != null &&
2017          Bytes.compareTo(hri.getStartKey(), splitPoint) == 0) {
2018        throw new IOException("should not give a splitkey which equals to startkey!");
2019     }
2020     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2021     ProtobufUtil.split(admin, hri, splitPoint);
2022   }
2023 
2024   /**
2025    * Modify an existing table, more IRB friendly version.
2026    * Asynchronous operation.  This means that it may be a while before your
2027    * schema change is updated across all of the table.
2028    *
2029    * @param tableName name of table.
2030    * @param htd modified description of the table
2031    * @throws IOException if a remote or network exception occurs
2032    */
2033   public void modifyTable(final TableName tableName, final HTableDescriptor htd)
2034   throws IOException {
2035     if (!tableName.equals(htd.getTableName())) {
2036       throw new IllegalArgumentException("the specified table name '" + tableName +
2037         "' doesn't match with the HTD one: " + htd.getTableName());
2038     }
2039 
2040     executeCallable(new MasterCallable<Void>(getConnection()) {
2041       @Override
2042       public Void call() throws ServiceException {
2043         ModifyTableRequest request = RequestConverter.buildModifyTableRequest(tableName, htd);
2044         master.modifyTable(null, request);
2045         return null;
2046       }
2047     });
2048   }
2049 
2050   public void modifyTable(final byte[] tableName, final HTableDescriptor htd)
2051   throws IOException {
2052     modifyTable(TableName.valueOf(tableName), htd);
2053   }
2054 
2055   public void modifyTable(final String tableName, final HTableDescriptor htd)
2056   throws IOException {
2057     modifyTable(TableName.valueOf(tableName), htd);
2058   }
2059 
2060   /**
2061    * @param tableNameOrRegionName Name of a table or name of a region.
2062    * @param ct A {@link CatalogTracker} instance (caller of this method usually has one).
2063    * @return a pair of HRegionInfo and ServerName if <code>tableNameOrRegionName</code> is
2064    *  a verified region name (we call {@link  MetaReader#getRegion( CatalogTracker, byte[])}
2065    *  else null.
2066    * Throw an exception if <code>tableNameOrRegionName</code> is null.
2067    * @throws IOException
2068    */
2069   Pair<HRegionInfo, ServerName> getRegion(final byte[] tableNameOrRegionName,
2070       final CatalogTracker ct) throws IOException {
2071     if (tableNameOrRegionName == null) {
2072       throw new IllegalArgumentException("Pass a table name or region name");
2073     }
2074     Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, tableNameOrRegionName);
2075     if (pair == null) {
2076       final AtomicReference<Pair<HRegionInfo, ServerName>> result =
2077         new AtomicReference<Pair<HRegionInfo, ServerName>>(null);
2078       final String encodedName = Bytes.toString(tableNameOrRegionName);
2079       MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
2080         @Override
2081         public boolean processRow(Result data) throws IOException {
2082           HRegionInfo info = HRegionInfo.getHRegionInfo(data);
2083           if (info == null) {
2084             LOG.warn("No serialized HRegionInfo in " + data);
2085             return true;
2086           }
2087           if (!encodedName.equals(info.getEncodedName())) return true;
2088           ServerName sn = HRegionInfo.getServerName(data);
2089           result.set(new Pair<HRegionInfo, ServerName>(info, sn));
2090           return false; // found the region, stop
2091         }
2092       };
2093 
2094       MetaScanner.metaScan(conf, connection, visitor, null);
2095       pair = result.get();
2096     }
2097     return pair;
2098   }
2099 
2100   /**
2101    * If the input is a region name, it is returned as is. If it's an
2102    * encoded region name, the corresponding region is found from meta
2103    * and its region name is returned. If we can't find any region in
2104    * meta matching the input as either region name or encoded region
2105    * name, the input is returned as is. We don't throw unknown
2106    * region exception.
2107    */
2108   private byte[] getRegionName(
2109       final byte[] regionNameOrEncodedRegionName) throws IOException {
2110     if (Bytes.equals(regionNameOrEncodedRegionName,
2111         HRegionInfo.FIRST_META_REGIONINFO.getRegionName())
2112           || Bytes.equals(regionNameOrEncodedRegionName,
2113             HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes())) {
2114       return HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
2115     }
2116     CatalogTracker ct = getCatalogTracker();
2117     byte[] tmp = regionNameOrEncodedRegionName;
2118     try {
2119       Pair<HRegionInfo, ServerName> regionServerPair
2120         = getRegion(regionNameOrEncodedRegionName, ct);
2121       if (regionServerPair != null && regionServerPair.getFirst() != null) {
2122         tmp = regionServerPair.getFirst().getRegionName();
2123       }
2124     } finally {
2125       cleanupCatalogTracker(ct);
2126     }
2127     return tmp;
2128   }
2129 
2130   /**
2131    * Check if table exists or not
2132    * @param tableName Name of a table.
2133    * @param ct A {@link CatalogTracker} instance (caller of this method usually has one).
2134    * @return tableName instance
2135    * @throws IOException if a remote or network exception occurs.
2136    * @throws TableNotFoundException if table does not exist.
2137    */
2138   //TODO rename this method
2139   private TableName checkTableExists(
2140       final TableName tableName, CatalogTracker ct)
2141       throws IOException {
2142     if (!MetaReader.tableExists(ct, tableName)) {
2143       throw new TableNotFoundException(tableName);
2144     }
2145     return tableName;
2146   }
2147 
2148   /**
2149    * Shuts down the HBase cluster
2150    * @throws IOException if a remote or network exception occurs
2151    */
2152   public synchronized void shutdown() throws IOException {
2153     executeCallable(new MasterCallable<Void>(getConnection()) {
2154       @Override
2155       public Void call() throws ServiceException {
2156         master.shutdown(null,ShutdownRequest.newBuilder().build());
2157         return null;
2158       }
2159     });
2160   }
2161 
2162   /**
2163    * Shuts down the current HBase master only.
2164    * Does not shutdown the cluster.
2165    * @see #shutdown()
2166    * @throws IOException if a remote or network exception occurs
2167    */
2168   public synchronized void stopMaster() throws IOException {
2169     executeCallable(new MasterCallable<Void>(getConnection()) {
2170       @Override
2171       public Void call() throws ServiceException {
2172         master.stopMaster(null,StopMasterRequest.newBuilder().build());
2173         return null;
2174       }
2175     });
2176   }
2177 
2178   /**
2179    * Stop the designated regionserver
2180    * @param hostnamePort Hostname and port delimited by a <code>:</code> as in
2181    * <code>example.org:1234</code>
2182    * @throws IOException if a remote or network exception occurs
2183    */
2184   public synchronized void stopRegionServer(final String hostnamePort)
2185   throws IOException {
2186     String hostname = Addressing.parseHostname(hostnamePort);
2187     int port = Addressing.parsePort(hostnamePort);
2188     AdminService.BlockingInterface admin =
2189       this.connection.getAdmin(ServerName.valueOf(hostname, port, 0));
2190     StopServerRequest request = RequestConverter.buildStopServerRequest(
2191       "Called by admin client " + this.connection.toString());
2192     try {
2193       admin.stopServer(null, request);
2194     } catch (ServiceException se) {
2195       throw ProtobufUtil.getRemoteException(se);
2196     }
2197   }
2198 
2199 
2200   /**
2201    * @return cluster status
2202    * @throws IOException if a remote or network exception occurs
2203    */
2204   public ClusterStatus getClusterStatus() throws IOException {
2205     return executeCallable(new MasterCallable<ClusterStatus>(getConnection()) {
2206       @Override
2207       public ClusterStatus call() throws ServiceException {
2208         GetClusterStatusRequest req = RequestConverter.buildGetClusterStatusRequest();
2209         return ClusterStatus.convert(master.getClusterStatus(null,req).getClusterStatus());
2210       }
2211     });
2212   }
2213 
2214   private HRegionLocation getFirstMetaServerForTable(final TableName tableName)
2215   throws IOException {
2216     return connection.locateRegion(TableName.META_TABLE_NAME,
2217       HRegionInfo.createRegionName(tableName, null, HConstants.NINES, false));
2218   }
2219 
2220   /**
2221    * @return Configuration used by the instance.
2222    */
2223   public Configuration getConfiguration() {
2224     return this.conf;
2225   }
2226 
2227   /**
2228    * Create a new namespace
2229    * @param descriptor descriptor which describes the new namespace
2230    * @throws IOException
2231    */
2232   public void createNamespace(final NamespaceDescriptor descriptor) throws IOException {
2233     executeCallable(new MasterCallable<Void>(getConnection()) {
2234       @Override
2235       public Void call() throws Exception {
2236         master.createNamespace(null,
2237           CreateNamespaceRequest.newBuilder()
2238                 .setNamespaceDescriptor(ProtobufUtil
2239                     .toProtoNamespaceDescriptor(descriptor)).build());
2240         return null;
2241       }
2242     });
2243   }
2244 
2245   /**
2246    * Modify an existing namespace
2247    * @param descriptor descriptor which describes the new namespace
2248    * @throws IOException
2249    */
2250   public void modifyNamespace(final NamespaceDescriptor descriptor) throws IOException {
2251     executeCallable(new MasterCallable<Void>(getConnection()) {
2252       @Override
2253       public Void call() throws Exception {
2254         master.modifyNamespace(null, ModifyNamespaceRequest.newBuilder().
2255           setNamespaceDescriptor(ProtobufUtil.toProtoNamespaceDescriptor(descriptor)).build());
2256         return null;
2257       }
2258     });
2259   }
2260 
2261   /**
2262    * Delete an existing namespace. Only empty namespaces (no tables) can be removed.
2263    * @param name namespace name
2264    * @throws IOException
2265    */
2266   public void deleteNamespace(final String name) throws IOException {
2267     executeCallable(new MasterCallable<Void>(getConnection()) {
2268       @Override
2269       public Void call() throws Exception {
2270         master.deleteNamespace(null, DeleteNamespaceRequest.newBuilder().
2271           setNamespaceName(name).build());
2272         return null;
2273       }
2274     });
2275   }
2276 
2277   /**
2278    * Get a namespace descriptor by name
2279    * @param name name of namespace descriptor
2280    * @return A descriptor
2281    * @throws IOException
2282    */
2283   public NamespaceDescriptor getNamespaceDescriptor(final String name) throws IOException {
2284     return
2285         executeCallable(new MasterCallable<NamespaceDescriptor>(getConnection()) {
2286           @Override
2287           public NamespaceDescriptor call() throws Exception {
2288             return ProtobufUtil.toNamespaceDescriptor(
2289               master.getNamespaceDescriptor(null, GetNamespaceDescriptorRequest.newBuilder().
2290                 setNamespaceName(name).build()).getNamespaceDescriptor());
2291           }
2292         });
2293   }
2294 
2295   /**
2296    * List available namespace descriptors
2297    * @return List of descriptors
2298    * @throws IOException
2299    */
2300   public NamespaceDescriptor[] listNamespaceDescriptors() throws IOException {
2301     return
2302         executeCallable(new MasterCallable<NamespaceDescriptor[]>(getConnection()) {
2303           @Override
2304           public NamespaceDescriptor[] call() throws Exception {
2305             List<HBaseProtos.NamespaceDescriptor> list =
2306               master.listNamespaceDescriptors(null, ListNamespaceDescriptorsRequest.newBuilder().
2307                 build()).getNamespaceDescriptorList();
2308             NamespaceDescriptor[] res = new NamespaceDescriptor[list.size()];
2309             for(int i = 0; i < list.size(); i++) {
2310               res[i] = ProtobufUtil.toNamespaceDescriptor(list.get(i));
2311             }
2312             return res;
2313           }
2314         });
2315   }
2316 
2317   /**
2318    * Get list of table descriptors by namespace
2319    * @param name namespace name
2320    * @return A descriptor
2321    * @throws IOException
2322    */
2323   public HTableDescriptor[] listTableDescriptorsByNamespace(final String name) throws IOException {
2324     return
2325         executeCallable(new MasterCallable<HTableDescriptor[]>(getConnection()) {
2326           @Override
2327           public HTableDescriptor[] call() throws Exception {
2328             List<TableSchema> list =
2329               master.listTableDescriptorsByNamespace(null, ListTableDescriptorsByNamespaceRequest.
2330                 newBuilder().setNamespaceName(name).build()).getTableSchemaList();
2331             HTableDescriptor[] res = new HTableDescriptor[list.size()];
2332             for(int i=0; i < list.size(); i++) {
2333 
2334               res[i] = HTableDescriptor.convert(list.get(i));
2335             }
2336             return res;
2337           }
2338         });
2339   }
2340 
2341   /**
2342    * Get list of table names by namespace
2343    * @param name namespace name
2344    * @return The list of table names in the namespace
2345    * @throws IOException
2346    */
2347   public TableName[] listTableNamesByNamespace(final String name) throws IOException {
2348     return
2349         executeCallable(new MasterCallable<TableName[]>(getConnection()) {
2350           @Override
2351           public TableName[] call() throws Exception {
2352             List<HBaseProtos.TableName> tableNames =
2353               master.listTableNamesByNamespace(null, ListTableNamesByNamespaceRequest.
2354                 newBuilder().setNamespaceName(name).build())
2355                 .getTableNameList();
2356             TableName[] result = new TableName[tableNames.size()];
2357             for (int i = 0; i < tableNames.size(); i++) {
2358               result[i] = ProtobufUtil.toTableName(tableNames.get(i));
2359             }
2360             return result;
2361           }
2362         });
2363   }
2364 
2365   /**
2366    * Check to see if HBase is running. Throw an exception if not.
2367    * We consider that HBase is running if ZooKeeper and Master are running.
2368    *
2369    * @param conf system configuration
2370    * @throws MasterNotRunningException if the master is not running
2371    * @throws ZooKeeperConnectionException if unable to connect to zookeeper
2372    */
2373   public static void checkHBaseAvailable(Configuration conf)
2374     throws MasterNotRunningException, ZooKeeperConnectionException, ServiceException, IOException {
2375     Configuration copyOfConf = HBaseConfiguration.create(conf);
2376 
2377     // We set it to make it fail as soon as possible if HBase is not available
2378     copyOfConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
2379     copyOfConf.setInt("zookeeper.recovery.retry", 0);
2380 
2381     HConnectionManager.HConnectionImplementation connection
2382       = (HConnectionManager.HConnectionImplementation)
2383       HConnectionManager.getConnection(copyOfConf);
2384 
2385     try {
2386       // Check ZK first.
2387       // If the connection exists, we may have a connection to ZK that does
2388       //  not work anymore
2389       ZooKeeperKeepAliveConnection zkw = null;
2390       try {
2391         zkw = connection.getKeepAliveZooKeeperWatcher();
2392         zkw.getRecoverableZooKeeper().getZooKeeper().exists(
2393           zkw.baseZNode, false);
2394 
2395       } catch (IOException e) {
2396         throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
2397       } catch (InterruptedException e) {
2398         throw (InterruptedIOException)
2399             new InterruptedIOException("Can't connect to ZooKeeper").initCause(e);
2400       } catch (KeeperException e) {
2401         throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
2402       } finally {
2403         if (zkw != null) {
2404           zkw.close();
2405         }
2406       }
2407 
2408       // Check Master
2409       connection.isMasterRunning();
2410 
2411     } finally {
2412       connection.close();
2413     }
2414   }
2415 
2416   /**
2417    * get the regions of a given table.
2418    *
2419    * @param tableName the name of the table
2420    * @return Ordered list of {@link HRegionInfo}.
2421    * @throws IOException
2422    */
2423   public List<HRegionInfo> getTableRegions(final TableName tableName)
2424   throws IOException {
2425     CatalogTracker ct = getCatalogTracker();
2426     List<HRegionInfo> Regions = null;
2427     try {
2428       Regions = MetaReader.getTableRegions(ct, tableName, true);
2429     } finally {
2430       cleanupCatalogTracker(ct);
2431     }
2432     return Regions;
2433   }
2434 
2435   public List<HRegionInfo> getTableRegions(final byte[] tableName)
2436   throws IOException {
2437     return getTableRegions(TableName.valueOf(tableName));
2438   }
2439 
2440   @Override
2441   public synchronized void close() throws IOException {
2442     if (cleanupConnectionOnClose && this.connection != null && !this.closed) {
2443       this.connection.close();
2444       this.closed = true;
2445     }
2446   }
2447 
2448   /**
2449    * Get tableDescriptors
2450    * @param tableNames List of table names
2451    * @return HTD[] the tableDescriptor
2452    * @throws IOException if a remote or network exception occurs
2453    */
2454   public HTableDescriptor[] getTableDescriptorsByTableName(List<TableName> tableNames)
2455   throws IOException {
2456     return this.connection.getHTableDescriptorsByTableName(tableNames);
2457   }
2458 
2459   /**
2460    * Get tableDescriptors
2461    * @param names List of table names
2462    * @return HTD[] the tableDescriptor
2463    * @throws IOException if a remote or network exception occurs
2464    */
2465   public HTableDescriptor[] getTableDescriptors(List<String> names)
2466   throws IOException {
2467     List<TableName> tableNames = new ArrayList<TableName>(names.size());
2468     for(String name : names) {
2469       tableNames.add(TableName.valueOf(name));
2470     }
2471     return getTableDescriptorsByTableName(tableNames);
2472   }
2473 
2474   /**
2475    * Roll the log writer. That is, start writing log messages to a new file.
2476    *
2477    * @param serverName
2478    *          The servername of the regionserver. A server name is made of host,
2479    *          port and startcode. This is mandatory. Here is an example:
2480    *          <code> host187.example.com,60020,1289493121758</code>
2481    * @return If lots of logs, flush the returned regions so next time through
2482    * we can clean logs. Returns null if nothing to flush.  Names are actual
2483    * region names as returned by {@link HRegionInfo#getEncodedName()}
2484    * @throws IOException if a remote or network exception occurs
2485    * @throws FailedLogCloseException
2486    */
2487  public synchronized  byte[][] rollHLogWriter(String serverName)
2488       throws IOException, FailedLogCloseException {
2489     ServerName sn = ServerName.valueOf(serverName);
2490     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2491     RollWALWriterRequest request = RequestConverter.buildRollWALWriterRequest();
2492     try {
2493       RollWALWriterResponse response = admin.rollWALWriter(null, request);
2494       int regionCount = response.getRegionToFlushCount();
2495       byte[][] regionsToFlush = new byte[regionCount][];
2496       for (int i = 0; i < regionCount; i++) {
2497         ByteString region = response.getRegionToFlush(i);
2498         regionsToFlush[i] = region.toByteArray();
2499       }
2500       return regionsToFlush;
2501     } catch (ServiceException se) {
2502       throw ProtobufUtil.getRemoteException(se);
2503     }
2504   }
2505 
2506   public String[] getMasterCoprocessors() {
2507     try {
2508       return getClusterStatus().getMasterCoprocessors();
2509     } catch (IOException e) {
2510       LOG.error("Could not getClusterStatus()",e);
2511       return null;
2512     }
2513   }
2514 
2515   /**
2516    * Get the current compaction state of a table or region.
2517    * It could be in a major compaction, a minor compaction, both, or none.
2518    *
2519    * @param tableNameOrRegionName table or region to major compact
2520    * @throws IOException if a remote or network exception occurs
2521    * @throws InterruptedException
2522    * @return the current compaction state
2523    */
2524   public CompactionState getCompactionState(final String tableNameOrRegionName)
2525       throws IOException, InterruptedException {
2526     return getCompactionState(Bytes.toBytes(tableNameOrRegionName));
2527   }
2528 
2529   /**
2530    * Get the current compaction state of a table or region.
2531    * It could be in a major compaction, a minor compaction, both, or none.
2532    *
2533    * @param tableNameOrRegionName table or region to major compact
2534    * @throws IOException if a remote or network exception occurs
2535    * @throws InterruptedException
2536    * @return the current compaction state
2537    */
2538   public CompactionState getCompactionState(final byte[] tableNameOrRegionName)
2539       throws IOException, InterruptedException {
2540     CompactionState state = CompactionState.NONE;
2541     CatalogTracker ct = getCatalogTracker();
2542     try {
2543       Pair<HRegionInfo, ServerName> regionServerPair
2544         = getRegion(tableNameOrRegionName, ct);
2545       if (regionServerPair != null) {
2546         if (regionServerPair.getSecond() == null) {
2547           throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
2548         } else {
2549           ServerName sn = regionServerPair.getSecond();
2550           AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2551           GetRegionInfoRequest request = RequestConverter.buildGetRegionInfoRequest(
2552             regionServerPair.getFirst().getRegionName(), true);
2553           GetRegionInfoResponse response = admin.getRegionInfo(null, request);
2554           return response.getCompactionState();
2555         }
2556       } else {
2557         final TableName tableName =
2558             checkTableExists(TableName.valueOf(tableNameOrRegionName), ct);
2559         List<Pair<HRegionInfo, ServerName>> pairs =
2560           MetaReader.getTableRegionsAndLocations(ct, tableName);
2561         for (Pair<HRegionInfo, ServerName> pair: pairs) {
2562           if (pair.getFirst().isOffline()) continue;
2563           if (pair.getSecond() == null) continue;
2564           try {
2565             ServerName sn = pair.getSecond();
2566             AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2567             GetRegionInfoRequest request = RequestConverter.buildGetRegionInfoRequest(
2568               pair.getFirst().getRegionName(), true);
2569             GetRegionInfoResponse response = admin.getRegionInfo(null, request);
2570             switch (response.getCompactionState()) {
2571             case MAJOR_AND_MINOR:
2572               return CompactionState.MAJOR_AND_MINOR;
2573             case MAJOR:
2574               if (state == CompactionState.MINOR) {
2575                 return CompactionState.MAJOR_AND_MINOR;
2576               }
2577               state = CompactionState.MAJOR;
2578               break;
2579             case MINOR:
2580               if (state == CompactionState.MAJOR) {
2581                 return CompactionState.MAJOR_AND_MINOR;
2582               }
2583               state = CompactionState.MINOR;
2584               break;
2585             case NONE:
2586               default: // nothing, continue
2587             }
2588           } catch (NotServingRegionException e) {
2589             if (LOG.isDebugEnabled()) {
2590               LOG.debug("Trying to get compaction state of " +
2591                 pair.getFirst() + ": " +
2592                 StringUtils.stringifyException(e));
2593             }
2594           } catch (RemoteException e) {
2595             if (e.getMessage().indexOf(NotServingRegionException.class.getName()) >= 0) {
2596               if (LOG.isDebugEnabled()) {
2597                 LOG.debug("Trying to get compaction state of " + pair.getFirst() + ": "
2598                     + StringUtils.stringifyException(e));
2599               }
2600             } else {
2601               throw e;
2602             }
2603           }
2604         }
2605       }
2606     } catch (ServiceException se) {
2607       throw ProtobufUtil.getRemoteException(se);
2608     } finally {
2609       cleanupCatalogTracker(ct);
2610     }
2611     return state;
2612   }
2613 
2614   /**
2615    * Take a snapshot for the given table. If the table is enabled, a FLUSH-type snapshot will be
2616    * taken. If the table is disabled, an offline snapshot is taken.
2617    * <p>
2618    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2619    * snapshot with the same name (even a different type or with different parameters) will fail with
2620    * a {@link SnapshotCreationException} indicating the duplicate naming.
2621    * <p>
2622    * Snapshot names follow the same naming constraints as tables in HBase. See
2623    * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
2624    * @param snapshotName name of the snapshot to be created
2625    * @param tableName name of the table for which snapshot is created
2626    * @throws IOException if a remote or network exception occurs
2627    * @throws SnapshotCreationException if snapshot creation failed
2628    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2629    */
2630   public void snapshot(final String snapshotName,
2631                        final TableName tableName) throws IOException,
2632       SnapshotCreationException, IllegalArgumentException {
2633     snapshot(snapshotName, tableName, SnapshotDescription.Type.FLUSH);
2634   }
2635 
2636   public void snapshot(final String snapshotName,
2637                        final String tableName) throws IOException,
2638       SnapshotCreationException, IllegalArgumentException {
2639     snapshot(snapshotName, TableName.valueOf(tableName),
2640         SnapshotDescription.Type.FLUSH);
2641   }
2642 
2643   /**
2644    * Create snapshot for the given table of given flush type.
2645    * <p>
2646    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2647    * snapshot with the same name (even a different type or with different parameters) will fail with
2648    * a {@link SnapshotCreationException} indicating the duplicate naming.
2649    * <p>
2650    * Snapshot names follow the same naming constraints as tables in HBase.
2651    * @param snapshotName name of the snapshot to be created
2652    * @param tableName name of the table for which snapshot is created
2653    * @param flushType if the snapshot should be taken without flush memstore first
2654    * @throws IOException if a remote or network exception occurs
2655    * @throws SnapshotCreationException if snapshot creation failed
2656    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2657    */
2658    public void snapshot(final byte[] snapshotName, final byte[] tableName,
2659                        final SnapshotDescription.Type flushType) throws
2660       IOException, SnapshotCreationException, IllegalArgumentException {
2661       snapshot(Bytes.toString(snapshotName), Bytes.toString(tableName), flushType);
2662   }
2663   /**
2664    public void snapshot(final String snapshotName,
2665     * Create a timestamp consistent snapshot for the given table.
2666                         final byte[] tableName) throws IOException,
2667     * <p>
2668     * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2669     * snapshot with the same name (even a different type or with different parameters) will fail with
2670     * a {@link SnapshotCreationException} indicating the duplicate naming.
2671     * <p>
2672     * Snapshot names follow the same naming constraints as tables in HBase.
2673     * @param snapshotName name of the snapshot to be created
2674     * @param tableName name of the table for which snapshot is created
2675     * @throws IOException if a remote or network exception occurs
2676     * @throws SnapshotCreationException if snapshot creation failed
2677     * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2678     */
2679   public void snapshot(final byte[] snapshotName,
2680                        final TableName tableName) throws IOException,
2681       SnapshotCreationException, IllegalArgumentException {
2682     snapshot(Bytes.toString(snapshotName), tableName, SnapshotDescription.Type.FLUSH);
2683   }
2684 
2685   public void snapshot(final byte[] snapshotName,
2686                        final byte[] tableName) throws IOException,
2687       SnapshotCreationException, IllegalArgumentException {
2688     snapshot(Bytes.toString(snapshotName), TableName.valueOf(tableName),
2689         SnapshotDescription.Type.FLUSH);
2690   }
2691 
2692   /**
2693    * Create typed snapshot of the table.
2694    * <p>
2695    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2696    * snapshot with the same name (even a different type or with different parameters) will fail with
2697    * a {@link SnapshotCreationException} indicating the duplicate naming.
2698    * <p>
2699    * Snapshot names follow the same naming constraints as tables in HBase. See
2700    * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
2701    * <p>
2702    * @param snapshotName name to give the snapshot on the filesystem. Must be unique from all other
2703    *          snapshots stored on the cluster
2704    * @param tableName name of the table to snapshot
2705    * @param type type of snapshot to take
2706    * @throws IOException we fail to reach the master
2707    * @throws SnapshotCreationException if snapshot creation failed
2708    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2709    */
2710   public void snapshot(final String snapshotName,
2711                        final TableName tableName,
2712                       SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
2713       IllegalArgumentException {
2714     SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
2715     builder.setTable(tableName.getNameAsString());
2716     builder.setName(snapshotName);
2717     builder.setType(type);
2718     snapshot(builder.build());
2719   }
2720 
2721   public void snapshot(final String snapshotName,
2722                        final String tableName,
2723                       SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
2724       IllegalArgumentException {
2725     snapshot(snapshotName, TableName.valueOf(tableName), type);
2726   }
2727 
2728   public void snapshot(final String snapshotName,
2729                        final byte[] tableName,
2730                       SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
2731       IllegalArgumentException {
2732     snapshot(snapshotName, TableName.valueOf(tableName), type);
2733   }
2734 
2735   /**
2736    * Take a snapshot and wait for the server to complete that snapshot (blocking).
2737    * <p>
2738    * Only a single snapshot should be taken at a time for an instance of HBase, or results may be
2739    * undefined (you can tell multiple HBase clusters to snapshot at the same time, but only one at a
2740    * time for a single cluster).
2741    * <p>
2742    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2743    * snapshot with the same name (even a different type or with different parameters) will fail with
2744    * a {@link SnapshotCreationException} indicating the duplicate naming.
2745    * <p>
2746    * Snapshot names follow the same naming constraints as tables in HBase. See
2747    * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
2748    * <p>
2749    * You should probably use {@link #snapshot(String, String)} or {@link #snapshot(byte[], byte[])}
2750    * unless you are sure about the type of snapshot that you want to take.
2751    * @param snapshot snapshot to take
2752    * @throws IOException or we lose contact with the master.
2753    * @throws SnapshotCreationException if snapshot failed to be taken
2754    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2755    */
2756   public void snapshot(SnapshotDescription snapshot) throws IOException, SnapshotCreationException,
2757       IllegalArgumentException {
2758     // actually take the snapshot
2759     SnapshotResponse response = takeSnapshotAsync(snapshot);
2760     final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot)
2761         .build();
2762     IsSnapshotDoneResponse done = null;
2763     long start = EnvironmentEdgeManager.currentTimeMillis();
2764     long max = response.getExpectedTimeout();
2765     long maxPauseTime = max / this.numRetries;
2766     int tries = 0;
2767     LOG.debug("Waiting a max of " + max + " ms for snapshot '" +
2768         ClientSnapshotDescriptionUtils.toString(snapshot) + "'' to complete. (max " +
2769         maxPauseTime + " ms per retry)");
2770     while (tries == 0
2771         || ((EnvironmentEdgeManager.currentTimeMillis() - start) < max && !done.getDone())) {
2772       try {
2773         // sleep a backoff <= pauseTime amount
2774         long sleep = getPauseTime(tries++);
2775         sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
2776         LOG.debug("(#" + tries + ") Sleeping: " + sleep +
2777           "ms while waiting for snapshot completion.");
2778         Thread.sleep(sleep);
2779 
2780       } catch (InterruptedException e) {
2781         LOG.debug("Interrupted while waiting for snapshot " + snapshot + " to complete");
2782         Thread.currentThread().interrupt();
2783       }
2784       LOG.debug("Getting current status of snapshot from master...");
2785       done = executeCallable(new MasterCallable<IsSnapshotDoneResponse>(getConnection()) {
2786         @Override
2787         public IsSnapshotDoneResponse call() throws ServiceException {
2788           return master.isSnapshotDone(null, request);
2789         }
2790       });
2791     };
2792     if (!done.getDone()) {
2793       throw new SnapshotCreationException("Snapshot '" + snapshot.getName()
2794           + "' wasn't completed in expectedTime:" + max + " ms", snapshot);
2795     }
2796   }
2797 
2798   /**
2799    * Take a snapshot without waiting for the server to complete that snapshot (asynchronous)
2800    * <p>
2801    * Only a single snapshot should be taken at a time, or results may be undefined.
2802    * @param snapshot snapshot to take
2803    * @return response from the server indicating the max time to wait for the snapshot
2804    * @throws IOException if the snapshot did not succeed or we lose contact with the master.
2805    * @throws SnapshotCreationException if snapshot creation failed
2806    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2807    */
2808   public SnapshotResponse takeSnapshotAsync(SnapshotDescription snapshot) throws IOException,
2809       SnapshotCreationException {
2810     ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);
2811     final SnapshotRequest request = SnapshotRequest.newBuilder().setSnapshot(snapshot)
2812         .build();
2813     // run the snapshot on the master
2814     return executeCallable(new MasterCallable<SnapshotResponse>(getConnection()) {
2815       @Override
2816       public SnapshotResponse call() throws ServiceException {
2817         return master.snapshot(null, request);
2818       }
2819     });
2820   }
2821 
2822   /**
2823    * Check the current state of the passed snapshot.
2824    * <p>
2825    * There are three possible states:
2826    * <ol>
2827    * <li>running - returns <tt>false</tt></li>
2828    * <li>finished - returns <tt>true</tt></li>
2829    * <li>finished with error - throws the exception that caused the snapshot to fail</li>
2830    * </ol>
2831    * <p>
2832    * The cluster only knows about the most recent snapshot. Therefore, if another snapshot has been
2833    * run/started since the snapshot your are checking, you will recieve an
2834    * {@link UnknownSnapshotException}.
2835    * @param snapshot description of the snapshot to check
2836    * @return <tt>true</tt> if the snapshot is completed, <tt>false</tt> if the snapshot is still
2837    *         running
2838    * @throws IOException if we have a network issue
2839    * @throws HBaseSnapshotException if the snapshot failed
2840    * @throws UnknownSnapshotException if the requested snapshot is unknown
2841    */
2842   public boolean isSnapshotFinished(final SnapshotDescription snapshot)
2843       throws IOException, HBaseSnapshotException, UnknownSnapshotException {
2844 
2845     return executeCallable(new MasterCallable<IsSnapshotDoneResponse>(getConnection()) {
2846       @Override
2847       public IsSnapshotDoneResponse call() throws ServiceException {
2848         return master.isSnapshotDone(null,
2849           IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot).build());
2850       }
2851     }).getDone();
2852   }
2853 
2854   /**
2855    * Restore the specified snapshot on the original table. (The table must be disabled)
2856    * If the "hbase.snapshot.restore.take.failsafe.snapshot" configuration property
2857    * is set to true, a snapshot of the current table is taken
2858    * before executing the restore operation.
2859    * In case of restore failure, the failsafe snapshot will be restored.
2860    * If the restore completes without problem the failsafe snapshot is deleted.
2861    *
2862    * @param snapshotName name of the snapshot to restore
2863    * @throws IOException if a remote or network exception occurs
2864    * @throws RestoreSnapshotException if snapshot failed to be restored
2865    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2866    */
2867   public void restoreSnapshot(final byte[] snapshotName)
2868       throws IOException, RestoreSnapshotException {
2869     restoreSnapshot(Bytes.toString(snapshotName));
2870   }
2871 
2872   /**
2873    * Restore the specified snapshot on the original table. (The table must be disabled)
2874    * If the "hbase.snapshot.restore.take.failsafe.snapshot" configuration property
2875    * is set to true, a snapshot of the current table is taken
2876    * before executing the restore operation.
2877    * In case of restore failure, the failsafe snapshot will be restored.
2878    * If the restore completes without problem the failsafe snapshot is deleted.
2879    *
2880    * @param snapshotName name of the snapshot to restore
2881    * @throws IOException if a remote or network exception occurs
2882    * @throws RestoreSnapshotException if snapshot failed to be restored
2883    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2884    */
2885   public void restoreSnapshot(final String snapshotName)
2886       throws IOException, RestoreSnapshotException {
2887     boolean takeFailSafeSnapshot =
2888       conf.getBoolean("hbase.snapshot.restore.take.failsafe.snapshot", false);
2889     restoreSnapshot(snapshotName, takeFailSafeSnapshot);
2890   }
2891 
2892   /**
2893    * Restore the specified snapshot on the original table. (The table must be disabled)
2894    * If 'takeFailSafeSnapshot' is set to true, a snapshot of the current table is taken
2895    * before executing the restore operation.
2896    * In case of restore failure, the failsafe snapshot will be restored.
2897    * If the restore completes without problem the failsafe snapshot is deleted.
2898    *
2899    * The failsafe snapshot name is configurable by using the property
2900    * "hbase.snapshot.restore.failsafe.name".
2901    *
2902    * @param snapshotName name of the snapshot to restore
2903    * @param takeFailSafeSnapshot true if the failsafe snapshot should be taken
2904    * @throws IOException if a remote or network exception occurs
2905    * @throws RestoreSnapshotException if snapshot failed to be restored
2906    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2907    */
2908   public void restoreSnapshot(final byte[] snapshotName, final boolean takeFailSafeSnapshot)
2909       throws IOException, RestoreSnapshotException {
2910     restoreSnapshot(Bytes.toString(snapshotName), takeFailSafeSnapshot);
2911   }
2912 
2913   /**
2914    * Restore the specified snapshot on the original table. (The table must be disabled)
2915    * If 'takeFailSafeSnapshot' is set to true, a snapshot of the current table is taken
2916    * before executing the restore operation.
2917    * In case of restore failure, the failsafe snapshot will be restored.
2918    * If the restore completes without problem the failsafe snapshot is deleted.
2919    *
2920    * The failsafe snapshot name is configurable by using the property
2921    * "hbase.snapshot.restore.failsafe.name".
2922    *
2923    * @param snapshotName name of the snapshot to restore
2924    * @param takeFailSafeSnapshot true if the failsafe snapshot should be taken
2925    * @throws IOException if a remote or network exception occurs
2926    * @throws RestoreSnapshotException if snapshot failed to be restored
2927    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2928    */
2929   public void restoreSnapshot(final String snapshotName, boolean takeFailSafeSnapshot)
2930       throws IOException, RestoreSnapshotException {
2931     TableName tableName = null;
2932     for (SnapshotDescription snapshotInfo: listSnapshots()) {
2933       if (snapshotInfo.getName().equals(snapshotName)) {
2934         tableName = TableName.valueOf(snapshotInfo.getTable());
2935         break;
2936       }
2937     }
2938 
2939     if (tableName == null) {
2940       throw new RestoreSnapshotException(
2941         "Unable to find the table name for snapshot=" + snapshotName);
2942     }
2943 
2944     // The table does not exists, switch to clone.
2945     if (!tableExists(tableName)) {
2946       try {
2947         cloneSnapshot(snapshotName, tableName);
2948       } catch (InterruptedException e) {
2949         throw new InterruptedIOException("Interrupted when restoring a nonexistent table: " +
2950           e.getMessage());
2951       }
2952       return;
2953     }
2954 
2955     // Check if the table is disabled
2956     if (!isTableDisabled(tableName)) {
2957       throw new TableNotDisabledException(tableName);
2958     }
2959 
2960     // Take a snapshot of the current state
2961     String failSafeSnapshotSnapshotName = null;
2962     if (takeFailSafeSnapshot) {
2963       failSafeSnapshotSnapshotName = conf.get("hbase.snapshot.restore.failsafe.name",
2964         "hbase-failsafe-{snapshot.name}-{restore.timestamp}");
2965       failSafeSnapshotSnapshotName = failSafeSnapshotSnapshotName
2966         .replace("{snapshot.name}", snapshotName)
2967         .replace("{table.name}", tableName.toString().replace(TableName.NAMESPACE_DELIM, '.'))
2968         .replace("{restore.timestamp}", String.valueOf(EnvironmentEdgeManager.currentTimeMillis()));
2969       LOG.info("Taking restore-failsafe snapshot: " + failSafeSnapshotSnapshotName);
2970       snapshot(failSafeSnapshotSnapshotName, tableName);
2971     }
2972 
2973     try {
2974       // Restore snapshot
2975       internalRestoreSnapshot(snapshotName, tableName);
2976     } catch (IOException e) {
2977       // Somthing went wrong during the restore...
2978       // if the pre-restore snapshot is available try to rollback
2979       if (takeFailSafeSnapshot) {
2980         try {
2981           internalRestoreSnapshot(failSafeSnapshotSnapshotName, tableName);
2982           String msg = "Restore snapshot=" + snapshotName +
2983             " failed. Rollback to snapshot=" + failSafeSnapshotSnapshotName + " succeeded.";
2984           LOG.error(msg, e);
2985           throw new RestoreSnapshotException(msg, e);
2986         } catch (IOException ex) {
2987           String msg = "Failed to restore and rollback to snapshot=" + failSafeSnapshotSnapshotName;
2988           LOG.error(msg, ex);
2989           throw new RestoreSnapshotException(msg, e);
2990         }
2991       } else {
2992         throw new RestoreSnapshotException("Failed to restore snapshot=" + snapshotName, e);
2993       }
2994     }
2995 
2996     // If the restore is succeeded, delete the pre-restore snapshot
2997     if (takeFailSafeSnapshot) {
2998       try {
2999         LOG.info("Deleting restore-failsafe snapshot: " + failSafeSnapshotSnapshotName);
3000         deleteSnapshot(failSafeSnapshotSnapshotName);
3001       } catch (IOException e) {
3002         LOG.error("Unable to remove the failsafe snapshot: " + failSafeSnapshotSnapshotName, e);
3003       }
3004     }
3005   }
3006 
3007   /**
3008    * Create a new table by cloning the snapshot content.
3009    *
3010    * @param snapshotName name of the snapshot to be cloned
3011    * @param tableName name of the table where the snapshot will be restored
3012    * @throws IOException if a remote or network exception occurs
3013    * @throws TableExistsException if table to be created already exists
3014    * @throws RestoreSnapshotException if snapshot failed to be cloned
3015    * @throws IllegalArgumentException if the specified table has not a valid name
3016    */
3017   public void cloneSnapshot(final byte[] snapshotName, final byte[] tableName)
3018       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
3019     cloneSnapshot(Bytes.toString(snapshotName), TableName.valueOf(tableName));
3020   }
3021 
3022   /**
3023    * Create a new table by cloning the snapshot content.
3024    *
3025    * @param snapshotName name of the snapshot to be cloned
3026    * @param tableName name of the table where the snapshot will be restored
3027    * @throws IOException if a remote or network exception occurs
3028    * @throws TableExistsException if table to be created already exists
3029    * @throws RestoreSnapshotException if snapshot failed to be cloned
3030    * @throws IllegalArgumentException if the specified table has not a valid name
3031    */
3032   public void cloneSnapshot(final byte[] snapshotName, final TableName tableName)
3033       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
3034     cloneSnapshot(Bytes.toString(snapshotName), tableName);
3035   }
3036 
3037 
3038 
3039   /**
3040    * Create a new table by cloning the snapshot content.
3041    *
3042    * @param snapshotName name of the snapshot to be cloned
3043    * @param tableName name of the table where the snapshot will be restored
3044    * @throws IOException if a remote or network exception occurs
3045    * @throws TableExistsException if table to be created already exists
3046    * @throws RestoreSnapshotException if snapshot failed to be cloned
3047    * @throws IllegalArgumentException if the specified table has not a valid name
3048    */
3049   public void cloneSnapshot(final String snapshotName, final String tableName)
3050       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
3051     cloneSnapshot(snapshotName, TableName.valueOf(tableName));
3052   }
3053 
3054   /**
3055    * Create a new table by cloning the snapshot content.
3056    *
3057    * @param snapshotName name of the snapshot to be cloned
3058    * @param tableName name of the table where the snapshot will be restored
3059    * @throws IOException if a remote or network exception occurs
3060    * @throws TableExistsException if table to be created already exists
3061    * @throws RestoreSnapshotException if snapshot failed to be cloned
3062    * @throws IllegalArgumentException if the specified table has not a valid name
3063    */
3064   public void cloneSnapshot(final String snapshotName, final TableName tableName)
3065       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
3066     if (tableExists(tableName)) {
3067       throw new TableExistsException(tableName);
3068     }
3069     internalRestoreSnapshot(snapshotName, tableName);
3070     waitUntilTableIsEnabled(tableName);
3071   }
3072 
3073   /**
3074    * Execute a distributed procedure on a cluster.
3075    *
3076    * @param signature A distributed procedure is uniquely identified
3077    * by its signature (default the root ZK node name of the procedure).
3078    * @param instance The instance name of the procedure. For some procedures, this parameter is
3079    * optional.
3080    * @param props Property/Value pairs of properties passing to the procedure
3081    */
3082   public void execProcedure(String signature, String instance,
3083       Map<String, String> props) throws IOException {
3084     ProcedureDescription.Builder builder = ProcedureDescription.newBuilder();
3085     builder.setSignature(signature).setInstance(instance);
3086     for (String key : props.keySet()) {
3087       NameStringPair pair = NameStringPair.newBuilder().setName(key)
3088           .setValue(props.get(key)).build();
3089       builder.addConfiguration(pair);
3090     }
3091 
3092     final ExecProcedureRequest request = ExecProcedureRequest.newBuilder()
3093         .setProcedure(builder.build()).build();
3094     // run the procedure on the master
3095     ExecProcedureResponse response = executeCallable(new MasterCallable<ExecProcedureResponse>(
3096         getConnection()) {
3097       @Override
3098       public ExecProcedureResponse call() throws ServiceException {
3099         return master.execProcedure(null, request);
3100       }
3101     });
3102 
3103     long start = EnvironmentEdgeManager.currentTimeMillis();
3104     long max = response.getExpectedTimeout();
3105     long maxPauseTime = max / this.numRetries;
3106     int tries = 0;
3107     LOG.debug("Waiting a max of " + max + " ms for procedure '" +
3108         signature + " : " + instance + "'' to complete. (max " + maxPauseTime + " ms per retry)");
3109     boolean done = false;
3110     while (tries == 0
3111         || ((EnvironmentEdgeManager.currentTimeMillis() - start) < max && !done)) {
3112       try {
3113         // sleep a backoff <= pauseTime amount
3114         long sleep = getPauseTime(tries++);
3115         sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
3116         LOG.debug("(#" + tries + ") Sleeping: " + sleep +
3117           "ms while waiting for procedure completion.");
3118         Thread.sleep(sleep);
3119 
3120       } catch (InterruptedException e) {
3121         LOG.debug("Interrupted while waiting for procedure " + signature + " to complete");
3122         Thread.currentThread().interrupt();
3123       }
3124       LOG.debug("Getting current status of procedure from master...");
3125       done = isProcedureFinished(signature, instance, props);
3126     }
3127     if (!done) {
3128       throw new IOException("Procedure '" + signature + " : " + instance
3129           + "' wasn't completed in expectedTime:" + max + " ms");
3130     }
3131   }
3132 
3133   /**
3134    * Check the current state of the specified procedure.
3135    * <p>
3136    * There are three possible states:
3137    * <ol>
3138    * <li>running - returns <tt>false</tt></li>
3139    * <li>finished - returns <tt>true</tt></li>
3140    * <li>finished with error - throws the exception that caused the procedure to fail</li>
3141    * </ol>
3142    * <p>
3143    *
3144    * @param signature The signature that uniquely identifies a procedure
3145    * @param instance The instance name of the procedure
3146    * @param props Property/Value pairs of properties passing to the procedure
3147    * @return true if the specified procedure is finished successfully, false if it is still running
3148    * @throws IOException if the specified procedure finished with error
3149    */
3150   public boolean isProcedureFinished(String signature, String instance, Map<String, String> props)
3151       throws IOException {
3152     final ProcedureDescription.Builder builder = ProcedureDescription.newBuilder();
3153     builder.setSignature(signature).setInstance(instance);
3154     for (String key : props.keySet()) {
3155       NameStringPair pair = NameStringPair.newBuilder().setName(key)
3156           .setValue(props.get(key)).build();
3157       builder.addConfiguration(pair);
3158     }
3159     final ProcedureDescription desc = builder.build();
3160     return executeCallable(
3161         new MasterCallable<IsProcedureDoneResponse>(getConnection()) {
3162           @Override
3163           public IsProcedureDoneResponse call() throws ServiceException {
3164             return master.isProcedureDone(null, IsProcedureDoneRequest
3165                 .newBuilder().setProcedure(desc).build());
3166           }
3167         }).getDone();
3168   }
3169 
3170   /**
3171    * Execute Restore/Clone snapshot and wait for the server to complete (blocking).
3172    * To check if the cloned table exists, use {@link #isTableAvailable} -- it is not safe to
3173    * create an HTable instance to this table before it is available.
3174    * @param snapshotName snapshot to restore
3175    * @param tableName table name to restore the snapshot on
3176    * @throws IOException if a remote or network exception occurs
3177    * @throws RestoreSnapshotException if snapshot failed to be restored
3178    * @throws IllegalArgumentException if the restore request is formatted incorrectly
3179    */
3180   private void internalRestoreSnapshot(final String snapshotName, final TableName
3181       tableName)
3182       throws IOException, RestoreSnapshotException {
3183     SnapshotDescription snapshot = SnapshotDescription.newBuilder()
3184         .setName(snapshotName).setTable(tableName.getNameAsString()).build();
3185 
3186     // actually restore the snapshot
3187     internalRestoreSnapshotAsync(snapshot);
3188 
3189     final IsRestoreSnapshotDoneRequest request = IsRestoreSnapshotDoneRequest.newBuilder()
3190         .setSnapshot(snapshot).build();
3191     IsRestoreSnapshotDoneResponse done = IsRestoreSnapshotDoneResponse.newBuilder()
3192         .setDone(false).buildPartial();
3193     final long maxPauseTime = 5000;
3194     int tries = 0;
3195     while (!done.getDone()) {
3196       try {
3197         // sleep a backoff <= pauseTime amount
3198         long sleep = getPauseTime(tries++);
3199         sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
3200         LOG.debug(tries + ") Sleeping: " + sleep + " ms while we wait for snapshot restore to complete.");
3201         Thread.sleep(sleep);
3202       } catch (InterruptedException e) {
3203         LOG.debug("Interrupted while waiting for snapshot " + snapshot + " restore to complete");
3204         Thread.currentThread().interrupt();
3205       }
3206       LOG.debug("Getting current status of snapshot restore from master...");
3207       done = executeCallable(new MasterCallable<IsRestoreSnapshotDoneResponse>(
3208           getConnection()) {
3209         @Override
3210         public IsRestoreSnapshotDoneResponse call() throws ServiceException {
3211           return master.isRestoreSnapshotDone(null, request);
3212         }
3213       });
3214     }
3215     if (!done.getDone()) {
3216       throw new RestoreSnapshotException("Snapshot '" + snapshot.getName() + "' wasn't restored.");
3217     }
3218   }
3219 
3220   /**
3221    * Execute Restore/Clone snapshot and wait for the server to complete (asynchronous)
3222    * <p>
3223    * Only a single snapshot should be restored at a time, or results may be undefined.
3224    * @param snapshot snapshot to restore
3225    * @return response from the server indicating the max time to wait for the snapshot
3226    * @throws IOException if a remote or network exception occurs
3227    * @throws RestoreSnapshotException if snapshot failed to be restored
3228    * @throws IllegalArgumentException if the restore request is formatted incorrectly
3229    */
3230   private RestoreSnapshotResponse internalRestoreSnapshotAsync(final SnapshotDescription snapshot)
3231       throws IOException, RestoreSnapshotException {
3232     ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);
3233 
3234     final RestoreSnapshotRequest request = RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot)
3235         .build();
3236 
3237     // run the snapshot restore on the master
3238     return executeCallable(new MasterCallable<RestoreSnapshotResponse>(getConnection()) {
3239       @Override
3240       public RestoreSnapshotResponse call() throws ServiceException {
3241         return master.restoreSnapshot(null, request);
3242       }
3243     });
3244   }
3245 
3246   /**
3247    * List completed snapshots.
3248    * @return a list of snapshot descriptors for completed snapshots
3249    * @throws IOException if a network error occurs
3250    */
3251   public List<SnapshotDescription> listSnapshots() throws IOException {
3252     return executeCallable(new MasterCallable<List<SnapshotDescription>>(getConnection()) {
3253       @Override
3254       public List<SnapshotDescription> call() throws ServiceException {
3255         return master.getCompletedSnapshots(null, GetCompletedSnapshotsRequest.newBuilder().build())
3256             .getSnapshotsList();
3257       }
3258     });
3259   }
3260 
3261   /**
3262    * List all the completed snapshots matching the given regular expression.
3263    *
3264    * @param regex The regular expression to match against
3265    * @return - returns a List of SnapshotDescription
3266    * @throws IOException if a remote or network exception occurs
3267    */
3268   public List<SnapshotDescription> listSnapshots(String regex) throws IOException {
3269     return listSnapshots(Pattern.compile(regex));
3270   }
3271 
3272   /**
3273    * List all the completed snapshots matching the given pattern.
3274    *
3275    * @param pattern The compiled regular expression to match against
3276    * @return - returns a List of SnapshotDescription
3277    * @throws IOException if a remote or network exception occurs
3278    */
3279   public List<SnapshotDescription> listSnapshots(Pattern pattern) throws IOException {
3280     List<SnapshotDescription> matched = new LinkedList<SnapshotDescription>();
3281     List<SnapshotDescription> snapshots = listSnapshots();
3282     for (SnapshotDescription snapshot : snapshots) {
3283       if (pattern.matcher(snapshot.getName()).matches()) {
3284         matched.add(snapshot);
3285       }
3286     }
3287     return matched;
3288   }
3289 
3290   /**
3291    * Delete an existing snapshot.
3292    * @param snapshotName name of the snapshot
3293    * @throws IOException if a remote or network exception occurs
3294    */
3295   public void deleteSnapshot(final byte[] snapshotName) throws IOException {
3296     deleteSnapshot(Bytes.toString(snapshotName));
3297   }
3298 
3299   /**
3300    * Delete an existing snapshot.
3301    * @param snapshotName name of the snapshot
3302    * @throws IOException if a remote or network exception occurs
3303    */
3304   public void deleteSnapshot(final String snapshotName) throws IOException {
3305     // make sure the snapshot is possibly valid
3306     TableName.isLegalFullyQualifiedTableName(Bytes.toBytes(snapshotName));
3307     // do the delete
3308     executeCallable(new MasterCallable<Void>(getConnection()) {
3309       @Override
3310       public Void call() throws ServiceException {
3311         master.deleteSnapshot(null,
3312           DeleteSnapshotRequest.newBuilder().
3313             setSnapshot(SnapshotDescription.newBuilder().setName(snapshotName).build()).build());
3314         return null;
3315       }
3316     });
3317   }
3318 
3319   /**
3320    * Delete existing snapshots whose names match the pattern passed.
3321    * @param regex The regular expression to match against
3322    * @throws IOException if a remote or network exception occurs
3323    */
3324   public void deleteSnapshots(final String regex) throws IOException {
3325     deleteSnapshots(Pattern.compile(regex));
3326   }
3327 
3328   /**
3329    * Delete existing snapshots whose names match the pattern passed.
3330    * @param pattern pattern for names of the snapshot to match
3331    * @throws IOException if a remote or network exception occurs
3332    */
3333   public void deleteSnapshots(final Pattern pattern) throws IOException {
3334     List<SnapshotDescription> snapshots = listSnapshots(pattern);
3335     for (final SnapshotDescription snapshot : snapshots) {
3336        try {
3337          internalDeleteSnapshot(snapshot);
3338        } catch (IOException ex) {
3339          LOG.info(
3340            "Failed to delete snapshot " + snapshot.getName() + " for table " + snapshot.getTable(),
3341            ex);
3342        }
3343     }
3344   }
3345 
3346   private void internalDeleteSnapshot(final SnapshotDescription snapshot) throws IOException {
3347     executeCallable(new MasterCallable<Void>(getConnection()) {
3348       @Override
3349       public Void call() throws ServiceException {
3350         this.master.deleteSnapshot(null, DeleteSnapshotRequest.newBuilder().setSnapshot(snapshot)
3351             .build());
3352         return null;
3353       }
3354     });
3355   }
3356 
3357   /**
3358    * Parent of {@link MasterCallable} and {@link MasterCallable}.
3359    * Has common methods.
3360    * @param <V>
3361    */
3362   abstract static class MasterCallable<V> implements RetryingCallable<V>, Closeable {
3363     protected HConnection connection;
3364     protected MasterKeepAliveConnection master;
3365 
3366     public MasterCallable(final HConnection connection) {
3367       this.connection = connection;
3368     }
3369 
3370     @Override
3371     public void prepare(boolean reload) throws IOException {
3372       this.master = this.connection.getKeepAliveMasterService();
3373     }
3374 
3375     @Override
3376     public void close() throws IOException {
3377       // The above prepare could fail but this would still be called though masterAdmin is null
3378       if (this.master != null) this.master.close();
3379     }
3380 
3381     @Override
3382     public void throwable(Throwable t, boolean retrying) {
3383     }
3384 
3385     @Override
3386     public String getExceptionMessageAdditionalDetail() {
3387       return "";
3388     }
3389 
3390     @Override
3391     public long sleep(long pause, int tries) {
3392       return ConnectionUtils.getPauseTime(pause, tries);
3393     }
3394   }
3395 
3396   private <V> V executeCallable(MasterCallable<V> callable) throws IOException {
3397     RpcRetryingCaller<V> caller = rpcCallerFactory.newCaller();
3398     try {
3399       return caller.callWithRetries(callable);
3400     } finally {
3401       callable.close();
3402     }
3403   }
3404 
3405   /**
3406    * Creates and returns a {@link com.google.protobuf.RpcChannel} instance
3407    * connected to the active master.
3408    *
3409    * <p>
3410    * The obtained {@link com.google.protobuf.RpcChannel} instance can be used to access a published
3411    * coprocessor {@link com.google.protobuf.Service} using standard protobuf service invocations:
3412    * </p>
3413    *
3414    * <div style="background-color: #cccccc; padding: 2px">
3415    * <blockquote><pre>
3416    * CoprocessorRpcChannel channel = myAdmin.coprocessorService();
3417    * MyService.BlockingInterface service = MyService.newBlockingStub(channel);
3418    * MyCallRequest request = MyCallRequest.newBuilder()
3419    *     ...
3420    *     .build();
3421    * MyCallResponse response = service.myCall(null, request);
3422    * </pre></blockquote></div>
3423    *
3424    * @return A MasterCoprocessorRpcChannel instance
3425    */
3426   public CoprocessorRpcChannel coprocessorService() {
3427     return new MasterCoprocessorRpcChannel(connection);
3428   }
3429 
3430   /**
3431    * Creates and returns a {@link com.google.protobuf.RpcChannel} instance
3432    * connected to the passed region server.
3433    *
3434    * <p>
3435    * The obtained {@link com.google.protobuf.RpcChannel} instance can be used to access a published
3436    * coprocessor {@link com.google.protobuf.Service} using standard protobuf service invocations:
3437    * </p>
3438    *
3439    * <div style="background-color: #cccccc; padding: 2px">
3440    * <blockquote><pre>
3441    * CoprocessorRpcChannel channel = myAdmin.coprocessorService(serverName);
3442    * MyService.BlockingInterface service = MyService.newBlockingStub(channel);
3443    * MyCallRequest request = MyCallRequest.newBuilder()
3444    *     ...
3445    *     .build();
3446    * MyCallResponse response = service.myCall(null, request);
3447    * </pre></blockquote></div>
3448    *
3449    * @param sn the server name to which the endpoint call is made
3450    * @return A RegionServerCoprocessorRpcChannel instance
3451    */
3452   public CoprocessorRpcChannel coprocessorService(ServerName sn) {
3453     return new RegionServerCoprocessorRpcChannel(connection, sn);
3454   }
3455 
3456   /**
3457    * Truncate a table. Synchronous operation.
3458    * @param tableName name of table to truncate
3459    * @param preserveSplits True if the splits should be preserved
3460    * @throws IOException if a remote or network exception occurs
3461    */
3462   public void truncateTable(final TableName tableName, final boolean preserveSplits)
3463       throws IOException {
3464     executeCallable(new MasterCallable<Void>(getConnection()) {
3465       @Override
3466       public Void call() throws ServiceException {
3467         LOG.info("Started truncate of " + tableName);
3468         TruncateTableRequest req = RequestConverter.buildTruncateTableRequest(
3469         tableName, preserveSplits);
3470         master.truncateTable(null, req);
3471         return null;
3472       }
3473     });
3474   }
3475 
3476 }