View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.catalog;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.net.ConnectException;
28  import java.util.concurrent.atomic.AtomicInteger;
29  
30  import junit.framework.Assert;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.hbase.Abortable;
36  import org.apache.hadoop.hbase.TableName;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.HRegionInfo;
40  import org.apache.hadoop.hbase.HRegionLocation;
41  import org.apache.hadoop.hbase.testclassification.MediumTests;
42  import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
43  import org.apache.hadoop.hbase.ServerName;
44  import org.apache.hadoop.hbase.client.HConnection;
45  import org.apache.hadoop.hbase.client.HConnectionManager;
46  import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
47  import org.apache.hadoop.hbase.client.Result;
48  import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
49  import org.apache.hadoop.hbase.master.RegionState;
50  import org.apache.hadoop.hbase.master.RegionState.State;
51  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
52  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
53  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
54  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
55  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
56  import org.apache.hadoop.hbase.util.Threads;
57  import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker;
58  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
59  import org.apache.hadoop.util.Progressable;
60  import org.apache.zookeeper.KeeperException;
61  import org.junit.After;
62  import org.junit.AfterClass;
63  import org.junit.Before;
64  import org.junit.BeforeClass;
65  import org.junit.Test;
66  import org.junit.experimental.categories.Category;
67  import org.mockito.Mockito;
68  
69  import com.google.protobuf.RpcController;
70  import com.google.protobuf.ServiceException;
71  
72  /**
73   * Test {@link CatalogTracker}
74   */
75  @Category(MediumTests.class)
76  public class TestCatalogTracker {
77    private static final Log LOG = LogFactory.getLog(TestCatalogTracker.class);
78    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
79    private static final ServerName SN =
80        ServerName.valueOf("example.org", 1234, System.currentTimeMillis());
81    private ZooKeeperWatcher watcher;
82    private Abortable abortable;
83  
84    @BeforeClass public static void beforeClass() throws Exception {
85      // Set this down so tests run quicker
86      UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
87      UTIL.startMiniZKCluster();
88    }
89  
90    @AfterClass public static void afterClass() throws IOException {
91      UTIL.getZkCluster().shutdown();
92    }
93  
94    @Before public void before() throws IOException {
95      this.abortable = new Abortable() {
96        @Override
97        public void abort(String why, Throwable e) {
98          LOG.info(why, e);
99        }
100 
101       @Override
102       public boolean isAborted()  {
103         return false;
104       }
105     };
106     this.watcher = new ZooKeeperWatcher(UTIL.getConfiguration(),
107       this.getClass().getSimpleName(), this.abortable, true);
108   }
109 
110   @After public void after() {
111     try {
112       // Clean out meta location or later tests will be confused... they presume
113       // start fresh in zk.
114       MetaRegionTracker.deleteMetaLocation(this.watcher);
115     } catch (KeeperException e) {
116       LOG.warn("Unable to delete hbase:meta location", e);
117     }
118 
119     // Clear out our doctored connection or could mess up subsequent tests.
120     HConnectionManager.deleteConnection(UTIL.getConfiguration());
121 
122     this.watcher.close();
123   }
124 
125   private CatalogTracker constructAndStartCatalogTracker(final HConnection c)
126   throws IOException, InterruptedException {
127     CatalogTracker ct = new CatalogTracker(this.watcher, UTIL.getConfiguration(),
128       c, this.abortable);
129     ct.start();
130     return ct;
131   }
132 
133   /**
134    * Test that we get notification if hbase:meta moves.
135    * @throws IOException
136    * @throws InterruptedException
137    * @throws KeeperException
138    */
139   @Test public void testThatIfMETAMovesWeAreNotified()
140   throws IOException, InterruptedException, KeeperException {
141     HConnection connection = Mockito.mock(HConnection.class);
142     constructAndStartCatalogTracker(connection);
143 
144     MetaRegionTracker.setMetaLocation(this.watcher,
145         ServerName.valueOf("example.com", 1234, System.currentTimeMillis()), State.OPEN);
146   }
147 
148   /**
149    * Test interruptable while blocking wait on meta.
150    * @throws IOException
151    * @throws ServiceException
152    * @throws InterruptedException
153    */
154   @Test public void testInterruptWaitOnMeta()
155   throws IOException, InterruptedException, ServiceException {
156     final ClientProtos.ClientService.BlockingInterface client =
157       Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
158     HConnection connection = mockConnection(null, client);
159 
160     Mockito.when(client.get((RpcController)Mockito.any(), (GetRequest)Mockito.any())).
161     thenReturn(GetResponse.newBuilder().build());
162     final CatalogTracker ct = constructAndStartCatalogTracker(connection);
163     ServerName meta = ct.getMetaLocation();
164     Assert.assertNull(meta);
165     Thread t = new Thread() {
166       @Override
167       public void run() {
168         try {
169           ct.waitForMeta();
170         } catch (InterruptedException e) {
171           throw new RuntimeException("Interrupted", e);
172         }
173       }
174     };
175     t.start();
176     while (!t.isAlive())
177       Threads.sleep(1);
178     Threads.sleep(1);
179     assertTrue(t.isAlive());
180     ct.stop();
181     // Join the thread... should exit shortly.
182     t.join();
183   }
184 
185   private void testVerifyMetaRegionLocationWithException(Exception ex)
186   throws IOException, InterruptedException, KeeperException, ServiceException {
187     // Mock an ClientProtocol.
188     final ClientProtos.ClientService.BlockingInterface implementation =
189       Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
190     HConnection connection = mockConnection(null, implementation);
191 
192     // If a 'get' is called on mocked interface, throw connection refused.
193     Mockito.when(implementation.get((RpcController) Mockito.any(), (GetRequest) Mockito.any())).
194       thenThrow(new ServiceException(ex));
195     // Now start up the catalogtracker with our doctored Connection.
196     final CatalogTracker ct = constructAndStartCatalogTracker(connection);
197 
198     MetaRegionTracker.setMetaLocation(this.watcher, SN, State.OPEN);
199     long timeout = UTIL.getConfiguration().
200       getLong("hbase.catalog.verification.timeout", 1000);
201     Assert.assertFalse(ct.verifyMetaRegionLocation(timeout));
202   }
203 
204   /**
205    * Test we survive a connection refused {@link ConnectException}
206    * @throws IOException
207    * @throws InterruptedException
208    * @throws KeeperException
209    * @throws ServiceException
210    */
211   @Test
212   public void testGetMetaServerConnectionFails()
213   throws IOException, InterruptedException, KeeperException, ServiceException {
214     testVerifyMetaRegionLocationWithException(new ConnectException("Connection refused"));
215   }
216 
217   /**
218    * Test that verifyMetaRegionLocation properly handles getting a
219    * ServerNotRunningException. See HBASE-4470.
220    * Note this doesn't check the exact exception thrown in the
221    * HBASE-4470 as there it is thrown from getHConnection() and
222    * here it is thrown from get() -- but those are both called
223    * from the same function anyway, and this way is less invasive than
224    * throwing from getHConnection would be.
225    *
226    * @throws IOException
227    * @throws InterruptedException
228    * @throws KeeperException
229    * @throws ServiceException
230    */
231   @Test
232   public void testVerifyMetaRegionServerNotRunning()
233   throws IOException, InterruptedException, KeeperException, ServiceException {
234     testVerifyMetaRegionLocationWithException(new ServerNotRunningYetException("mock"));
235   }
236 
237   /**
238    * Test get of meta region fails properly if nothing to connect to.
239    * @throws IOException
240    * @throws InterruptedException
241    * @throws KeeperException
242    * @throws ServiceException
243    */
244   @Test
245   public void testVerifyMetaRegionLocationFails()
246   throws IOException, InterruptedException, KeeperException, ServiceException {
247     HConnection connection = Mockito.mock(HConnection.class);
248     ServiceException connectException =
249       new ServiceException(new ConnectException("Connection refused"));
250     final AdminProtos.AdminService.BlockingInterface implementation =
251       Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
252     Mockito.when(implementation.getRegionInfo((RpcController)Mockito.any(),
253       (GetRegionInfoRequest)Mockito.any())).thenThrow(connectException);
254     Mockito.when(connection.getAdmin(Mockito.any(ServerName.class), Mockito.anyBoolean())).
255       thenReturn(implementation);
256     final CatalogTracker ct = constructAndStartCatalogTracker(connection);
257 
258     MetaRegionTracker.setMetaLocation(this.watcher,
259         ServerName.valueOf("example.com", 1234, System.currentTimeMillis()), State.OPEN);
260     Assert.assertFalse(ct.verifyMetaRegionLocation(100));
261   }
262 
263   @Test (expected = NotAllMetaRegionsOnlineException.class)
264   public void testTimeoutWaitForMeta()
265   throws IOException, InterruptedException {
266     HConnection connection = Mockito.mock(HConnection.class);
267     final CatalogTracker ct = constructAndStartCatalogTracker(connection);
268     ct.waitForMeta(100);
269   }
270 
271   /**
272    * Test waiting on meat w/ no timeout specified.
273    * @throws IOException
274    * @throws InterruptedException
275    * @throws KeeperException
276    */
277   @Test public void testNoTimeoutWaitForMeta()
278   throws IOException, InterruptedException, KeeperException {
279     HConnection connection = Mockito.mock(HConnection.class);
280     final CatalogTracker ct = constructAndStartCatalogTracker(connection);
281     ServerName hsa = ct.getMetaLocation();
282     Assert.assertNull(hsa);
283 
284     // Now test waiting on meta location getting set.
285     Thread t = new WaitOnMetaThread(ct);
286     startWaitAliveThenWaitItLives(t, 1);
287     // Set a meta location.
288     hsa = setMetaLocation();
289     // Join the thread... should exit shortly.
290     t.join();
291     // Now meta is available.
292     Assert.assertTrue(ct.getMetaLocation().equals(hsa));
293   }
294   
295   
296   /**
297    * Test normal operations
298    */
299   @Test 
300   public void testMetaLookup()
301           throws IOException, InterruptedException, ServiceException, KeeperException {
302     HConnection connection = Mockito.mock(HConnection.class);
303     final CatalogTracker ct = constructAndStartCatalogTracker(connection);
304     
305     assertNull(ct.getMetaLocation());
306     for (RegionState.State state : RegionState.State.values()) {
307       if (state.equals(RegionState.State.OPEN))
308         continue;
309       MetaRegionTracker.setMetaLocation(this.watcher, SN, state);
310       assertNull(ct.getMetaLocation());
311       assertEquals(state, ct.getMetaRegionState().getState());
312     }
313     MetaRegionTracker.setMetaLocation(this.watcher, SN, RegionState.State.OPEN);
314     assertEquals(ct.getMetaLocation(), SN);
315     assertEquals(RegionState.State.OPEN,
316       ct.getMetaRegionState().getState());
317 
318     MetaRegionTracker.deleteMetaLocation(this.watcher);
319     assertNull(ct.getMetaRegionState().getServerName());
320     assertEquals(ct.getMetaRegionState().getState(),
321       RegionState.State.OFFLINE);
322     assertNull(ct.getMetaLocation());
323   }
324   
325   private ServerName setMetaLocation() throws KeeperException {
326     MetaRegionTracker.setMetaLocation(this.watcher, SN, State.OPEN);
327     return SN;
328   }
329 
330   /**
331    * @param admin An {@link AdminProtos.AdminService.BlockingInterface} instance; you'll likely
332    * want to pass a mocked HRS; can be null.
333    * @param client A mocked ClientProtocol instance, can be null
334    * @return Mock up a connection that returns a {@link Configuration} when
335    * {@link HConnection#getConfiguration()} is called, a 'location' when
336    * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called,
337    * and that returns the passed {@link AdminProtos.AdminService.BlockingInterface} instance when
338    * {@link HConnection#getAdmin(ServerName)} is called, returns the passed
339    * {@link ClientProtos.ClientService.BlockingInterface} instance when
340    * {@link HConnection#getClient(ServerName)} is called (Be sure to call
341    * {@link HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration)}
342    * when done with this mocked Connection.
343    * @throws IOException
344    */
345   private HConnection mockConnection(final AdminProtos.AdminService.BlockingInterface admin,
346       final ClientProtos.ClientService.BlockingInterface client)
347   throws IOException {
348     HConnection connection =
349       HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
350     Mockito.doNothing().when(connection).close();
351     // Make it so we return any old location when asked.
352     final HRegionLocation anyLocation =
353       new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN);
354     Mockito.when(connection.getRegionLocation((TableName) Mockito.any(),
355         (byte[]) Mockito.any(), Mockito.anyBoolean())).
356       thenReturn(anyLocation);
357     Mockito.when(connection.locateRegion((TableName) Mockito.any(),
358         (byte[]) Mockito.any())).
359       thenReturn(anyLocation);
360     if (admin != null) {
361       // If a call to getHRegionConnection, return this implementation.
362       Mockito.when(connection.getAdmin(Mockito.any(ServerName.class))).
363         thenReturn(admin);
364     }
365     if (client != null) {
366       // If a call to getClient, return this implementation.
367       Mockito.when(connection.getClient(Mockito.any(ServerName.class))).
368         thenReturn(client);
369     }
370     return connection;
371   }
372 
373   /**
374    * @return A mocked up Result that fakes a Get on a row in the
375    * <code>hbase:meta</code> table.
376    * @throws IOException
377    */
378   private Result getMetaTableRowResult() throws IOException {
379     return MetaMockingUtil.getMetaTableRowResult(HRegionInfo.FIRST_META_REGIONINFO, SN);
380   }
381 
382   private void startWaitAliveThenWaitItLives(final Thread t, final int ms) {
383     t.start();
384     while(!t.isAlive()) {
385       // Wait
386     }
387     // Wait one second.
388     Threads.sleep(ms);
389     Assert.assertTrue("Assert " + t.getName() + " still waiting", t.isAlive());
390   }
391 
392   class CountingProgressable implements Progressable {
393     final AtomicInteger counter = new AtomicInteger(0);
394     @Override
395     public void progress() {
396       this.counter.incrementAndGet();
397     }
398   }
399 
400   /**
401    * Wait on META.
402    */
403   class WaitOnMetaThread extends Thread {
404     final CatalogTracker ct;
405 
406     WaitOnMetaThread(final CatalogTracker ct) {
407       super("WaitOnMeta");
408       this.ct = ct;
409     }
410 
411     @Override
412     public void run() {
413       try {
414         doWaiting();
415       } catch (InterruptedException e) {
416         throw new RuntimeException("Failed wait", e);
417       }
418       LOG.info("Exiting " + getName());
419     }
420 
421     void doWaiting() throws InterruptedException {
422       try {
423         while (this.ct.waitForMeta(10000) == null);
424       } catch (NotAllMetaRegionsOnlineException e) {
425         // Ignore.
426       }
427     }
428   }
429 
430 }