View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import org.apache.commons.logging.Log;
21  import org.apache.commons.logging.LogFactory;
22  import org.apache.hadoop.conf.Configuration;
23  import org.apache.hadoop.hbase.*;
24  import org.apache.hadoop.hbase.client.*;
25  import org.apache.hadoop.hbase.test.MetricsAssertHelper;
26  import org.apache.hadoop.hbase.testclassification.MediumTests;
27  import org.apache.hadoop.hbase.util.Bytes;
28  import org.apache.hadoop.hbase.util.Threads;
29  import org.apache.log4j.Level;
30  import org.apache.log4j.Logger;
31  import org.junit.AfterClass;
32  import org.junit.BeforeClass;
33  import org.junit.Test;
34  import org.junit.experimental.categories.Category;
35  import static org.junit.Assert.*;
36  
37  import java.io.IOException;
38  import java.util.ArrayList;
39  import java.util.List;
40  
41  
42  @Category(MediumTests.class)
43  public class TestRegionServerMetrics {
44    private static final Log LOG = LogFactory.getLog(TestRegionServerMetrics.class);
45    private static MetricsAssertHelper metricsHelper;
46  
47    static {
48      Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG);
49    }
50  
51    private static MiniHBaseCluster cluster;
52    private static HRegionServer rs;
53    private static Configuration conf;
54    private static HBaseTestingUtility TEST_UTIL;
55    private static MetricsRegionServer metricsRegionServer;
56    private static MetricsRegionServerSource serverSource;
57  
58    @BeforeClass
59    public static void startCluster() throws Exception {
60      metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class);
61      TEST_UTIL = new HBaseTestingUtility();
62      conf = TEST_UTIL.getConfiguration();
63      conf.getLong("hbase.splitlog.max.resubmit", 0);
64      // Make the failure test faster
65      conf.setInt("zookeeper.recovery.retry", 0);
66      conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
67  
68      TEST_UTIL.startMiniCluster(1, 1);
69      cluster = TEST_UTIL.getHBaseCluster();
70  
71      cluster.waitForActiveAndReadyMaster();
72  
73      while (cluster.getLiveRegionServerThreads().size() < 1) {
74        Threads.sleep(100);
75      }
76  
77      rs = cluster.getRegionServer(0);
78      metricsRegionServer = rs.getMetrics();
79      serverSource = metricsRegionServer.getMetricsSource();
80    }
81  
82    @AfterClass
83    public static void after() throws Exception {
84      if (TEST_UTIL != null) {
85        TEST_UTIL.shutdownMiniCluster();
86      }
87    }
88  
89    @Test(timeout = 300000)
90    public void testRegionCount() throws Exception {
91      String regionMetricsKey = "regionCount";
92      long regions = metricsHelper.getGaugeLong(regionMetricsKey, serverSource);
93      // Creating a table should add one region
94      TEST_UTIL.createTable(Bytes.toBytes("table"), Bytes.toBytes("cf"));
95      metricsHelper.assertGaugeGt(regionMetricsKey, regions, serverSource);
96    }
97  
98    @Test
99    public void testLocalFiles() throws Exception {
100     metricsHelper.assertGauge("percentFilesLocal", 0, serverSource);
101   }
102 
103   @Test
104   public void testRequestCount() throws Exception {
105     String tableNameString = "testRequestCount";
106     byte[] tName = Bytes.toBytes(tableNameString);
107     byte[] cfName = Bytes.toBytes("d");
108     byte[] row = Bytes.toBytes("rk");
109     byte[] qualifier = Bytes.toBytes("qual");
110     byte[] initValue = Bytes.toBytes("Value");
111     byte[] nextValue = Bytes.toBytes("NEXT VAL");
112 
113 
114     TEST_UTIL.createTable(tName, cfName);
115 
116     new HTable(conf, tName).close(); //wait for the table to come up.
117 
118     // Do a first put to be sure that the connection is established, meta is there and so on.
119     HTable table = new HTable(conf, tName);
120     Put p = new Put(row);
121     p.add(cfName, qualifier, initValue);
122     table.put(p);
123 
124     metricsRegionServer.getRegionServerWrapper().forceRecompute();
125     long requests = metricsHelper.getCounter("totalRequestCount", serverSource);
126     long readRequests = metricsHelper.getCounter("readRequestCount", serverSource);
127     long writeRequests = metricsHelper.getCounter("writeRequestCount", serverSource);
128 
129     for (int i=0; i< 30; i++) {
130       table.put(p);
131     }
132 
133     metricsRegionServer.getRegionServerWrapper().forceRecompute();
134     metricsHelper.assertCounter("totalRequestCount", requests + 30, serverSource);
135     metricsHelper.assertCounter("readRequestCount", readRequests, serverSource);
136     metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
137 
138     Get g = new Get(row);
139     for (int i=0; i< 10; i++) {
140       table.get(g);
141     }
142 
143     metricsRegionServer.getRegionServerWrapper().forceRecompute();
144     metricsHelper.assertCounter("totalRequestCount", requests + 40, serverSource);
145     metricsHelper.assertCounter("readRequestCount", readRequests + 10, serverSource);
146     metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
147 
148     for ( HRegionInfo i:table.getRegionLocations().keySet()) {
149       MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
150           .getMetrics()
151           .getSource()
152           .getAggregateSource();
153       String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
154           "_table_"+tableNameString +
155           "_region_" + i.getEncodedName()+
156           "_metric";
157       metricsHelper.assertCounter(prefix + "_getNumOps", 10, agg);
158       metricsHelper.assertCounter(prefix + "_mutateCount", 31, agg);
159     }
160 
161     // 0.98 specific; the loop above does reads. The 0.99 does not
162     metricsRegionServer.getRegionServerWrapper().forceRecompute();
163     metricsHelper.assertCounter("totalRequestCount", requests + 40 + 3, serverSource);
164     metricsHelper.assertCounter("readRequestCount", readRequests + 10 + 1, serverSource);
165     // end of 0.98 specific
166 
167     List<Get> gets = new ArrayList<Get>();
168     for (int i=0; i< 10; i++) {
169       gets.add(new Get(row));
170     }
171     table.get(gets);
172 
173     metricsRegionServer.getRegionServerWrapper().forceRecompute();
174     metricsHelper.assertCounter("totalRequestCount", requests + 50 + 3, serverSource);
175     metricsHelper.assertCounter("readRequestCount", readRequests + 20 + 1, serverSource);
176     metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
177 
178     table.setAutoFlushTo(false);
179     for (int i=0; i< 30; i++) {
180       table.put(p);
181     }
182     table.flushCommits();
183 
184     metricsRegionServer.getRegionServerWrapper().forceRecompute();
185     metricsHelper.assertCounter("totalRequestCount", requests + 80 + 3, serverSource);
186     metricsHelper.assertCounter("readRequestCount", readRequests + 20 + 1, serverSource);
187     metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource);
188 
189     table.close();
190   }
191 
192   @Test
193   public void testMutationsWithoutWal() throws Exception {
194     byte[] tableName = Bytes.toBytes("testMutationsWithoutWal");
195     byte[] cf = Bytes.toBytes("d");
196     byte[] row = Bytes.toBytes("rk");
197     byte[] qualifier = Bytes.toBytes("qual");
198     byte[] val = Bytes.toBytes("Value");
199 
200     metricsRegionServer.getRegionServerWrapper().forceRecompute();
201 
202     TEST_UTIL.createTable(tableName, cf);
203 
204     HTable t = new HTable(conf, tableName);
205 
206     Put p = new Put(row);
207     p.add(cf, qualifier, val);
208     p.setDurability(Durability.SKIP_WAL);
209 
210     t.put(p);
211     t.flushCommits();
212 
213     metricsRegionServer.getRegionServerWrapper().forceRecompute();
214     metricsHelper.assertGauge("mutationsWithoutWALCount", 1, serverSource);
215     long minLength = row.length + cf.length + qualifier.length + val.length;
216     metricsHelper.assertGaugeGt("mutationsWithoutWALSize", minLength, serverSource);
217 
218     t.close();
219   }
220 
221   @Test
222   public void testStoreCount() throws Exception {
223     byte[] tableName = Bytes.toBytes("testStoreCount");
224     byte[] cf = Bytes.toBytes("d");
225     byte[] row = Bytes.toBytes("rk");
226     byte[] qualifier = Bytes.toBytes("qual");
227     byte[] val = Bytes.toBytes("Value");
228 
229     metricsRegionServer.getRegionServerWrapper().forceRecompute();
230     long stores = metricsHelper.getGaugeLong("storeCount", serverSource);
231     long storeFiles = metricsHelper.getGaugeLong("storeFileCount", serverSource);
232 
233     TEST_UTIL.createTable(tableName, cf);
234 
235     //Force a hfile.
236     HTable t = new HTable(conf, tableName);
237     Put p = new Put(row);
238     p.add(cf, qualifier, val);
239     t.put(p);
240     t.flushCommits();
241     TEST_UTIL.getHBaseAdmin().flush(tableName);
242 
243     metricsRegionServer.getRegionServerWrapper().forceRecompute();
244     metricsHelper.assertGauge("storeCount", stores +1, serverSource);
245     metricsHelper.assertGauge("storeFileCount", storeFiles + 1, serverSource);
246 
247     t.close();
248   }
249 
250   @Test
251   public void testCheckAndPutCount() throws Exception {
252     String tableNameString = "testCheckAndPutCount";
253     byte[] tableName = Bytes.toBytes(tableNameString);
254     byte[] cf = Bytes.toBytes("d");
255     byte[] row = Bytes.toBytes("rk");
256     byte[] qualifier = Bytes.toBytes("qual");
257     byte[] valOne = Bytes.toBytes("Value");
258     byte[] valTwo = Bytes.toBytes("ValueTwo");
259     byte[] valThree = Bytes.toBytes("ValueThree");
260 
261     TEST_UTIL.createTable(tableName, cf);
262     HTable t = new HTable(conf, tableName);
263     Put p = new Put(row);
264     p.add(cf, qualifier, valOne);
265     t.put(p);
266     t.flushCommits();
267 
268     Put pTwo = new Put(row);
269     pTwo.add(cf, qualifier, valTwo);
270     t.checkAndPut(row, cf, qualifier, valOne, pTwo);
271     t.flushCommits();
272 
273     Put pThree = new Put(row);
274     pThree.add(cf, qualifier, valThree);
275     t.checkAndPut(row, cf, qualifier, valOne, pThree);
276     t.flushCommits();
277 
278 
279     metricsRegionServer.getRegionServerWrapper().forceRecompute();
280     metricsHelper.assertCounter("checkMutateFailedCount", 1, serverSource);
281     metricsHelper.assertCounter("checkMutatePassedCount", 1, serverSource);
282 
283     t.close();
284   }
285 
286   @Test
287   public void testIncrement() throws Exception {
288     String tableNameString = "testIncrement";
289     byte[] tableName = Bytes.toBytes(tableNameString);
290     byte[] cf = Bytes.toBytes("d");
291     byte[] row = Bytes.toBytes("rk");
292     byte[] qualifier = Bytes.toBytes("qual");
293     byte[] val = Bytes.toBytes(0l);
294 
295 
296     TEST_UTIL.createTable(tableName, cf);
297     HTable t = new HTable(conf, tableName);
298 
299     Put p = new Put(row);
300     p.add(cf, qualifier, val);
301     t.put(p);
302     t.flushCommits();
303 
304     for(int count = 0; count< 13; count++) {
305       Increment inc = new Increment(row);
306       inc.addColumn(cf, qualifier, 100);
307       t.increment(inc);
308     }
309 
310     t.flushCommits();
311 
312     metricsRegionServer.getRegionServerWrapper().forceRecompute();
313     metricsHelper.assertCounter("incrementNumOps", 13, serverSource);
314 
315     t.close();
316   }
317 
318   @Test
319   public void testAppend() throws Exception {
320     String tableNameString = "testAppend";
321     byte[] tableName = Bytes.toBytes(tableNameString);
322     byte[] cf = Bytes.toBytes("d");
323     byte[] row = Bytes.toBytes("rk");
324     byte[] qualifier = Bytes.toBytes("qual");
325     byte[] val = Bytes.toBytes("One");
326 
327 
328     TEST_UTIL.createTable(tableName, cf);
329     HTable t = new HTable(conf, tableName);
330 
331     Put p = new Put(row);
332     p.add(cf, qualifier, val);
333     t.put(p);
334     t.flushCommits();
335 
336     for(int count = 0; count< 73; count++) {
337       Append append = new Append(row);
338       append.add(cf, qualifier, Bytes.toBytes(",Test"));
339       t.append(append);
340     }
341 
342     t.flushCommits();
343 
344     metricsRegionServer.getRegionServerWrapper().forceRecompute();
345     metricsHelper.assertCounter("appendNumOps", 73, serverSource);
346 
347     t.close();
348   }
349 
350   @Test
351   public void testScanNext() throws IOException {
352     String tableNameString = "testScanNext";
353     byte[] tableName = Bytes.toBytes(tableNameString);
354     byte[] cf = Bytes.toBytes("d");
355     byte[] qualifier = Bytes.toBytes("qual");
356     byte[] val = Bytes.toBytes("One");
357 
358 
359     TEST_UTIL.createTable(tableName, cf);
360     HTable t = new HTable(conf, tableName);
361     t.setAutoFlush(false, true);
362     for (int insertCount =0; insertCount < 100; insertCount++) {
363       Put p = new Put(Bytes.toBytes("" + insertCount + "row"));
364       p.add(cf, qualifier, val);
365       t.put(p);
366     }
367     t.flushCommits();
368 
369     Scan s = new Scan();
370     s.setBatch(1);
371     s.setCaching(1);
372     ResultScanner resultScanners = t.getScanner(s);
373 
374     for (int nextCount = 0; nextCount < 30; nextCount++) {
375       Result result = resultScanners.next();
376       assertNotNull(result);
377       assertEquals(1, result.size());
378     }
379     for ( HRegionInfo i:t.getRegionLocations().keySet()) {
380       MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
381           .getMetrics()
382           .getSource()
383           .getAggregateSource();
384       String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
385           "_table_"+tableNameString +
386           "_region_" + i.getEncodedName()+
387           "_metric";
388       metricsHelper.assertCounter(prefix + "_scanNextNumOps", 30, agg);
389     }
390   }
391 }