1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import static org.junit.Assert.*;
22
23 import java.io.IOException;
24 import java.util.List;
25 import java.util.ArrayList;
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.hbase.client.HTable;
30 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
31 import org.apache.hadoop.hbase.regionserver.HRegionServer;
32 import org.apache.hadoop.hbase.regionserver.HRegion;
33 import org.apache.hadoop.hbase.testclassification.MediumTests;
34 import org.apache.hadoop.hbase.util.Bytes;
35 import org.apache.hadoop.hbase.util.JVMClusterUtil;
36 import org.apache.hadoop.hbase.util.Threads;
37 import org.junit.Test;
38 import org.junit.experimental.categories.Category;
39
40
41
42
43
44 @Category(MediumTests.class)
45 public class TestGlobalMemStoreSize {
46 private final Log LOG = LogFactory.getLog(this.getClass().getName());
47 private static int regionServerNum = 4;
48 private static int regionNum = 16;
49
50 private static int totalRegionNum = regionNum+2;
51
52 private HBaseTestingUtility TEST_UTIL;
53 private MiniHBaseCluster cluster;
54
55
56
57
58
59
60 @Test
61 public void testGlobalMemStore() throws Exception {
62
63 LOG.info("Starting cluster");
64 Configuration conf = HBaseConfiguration.create();
65 conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000);
66 conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 5000);
67 TEST_UTIL = new HBaseTestingUtility(conf);
68 TEST_UTIL.startMiniCluster(1, regionServerNum);
69 cluster = TEST_UTIL.getHBaseCluster();
70 LOG.info("Waiting for active/ready master");
71 cluster.waitForActiveAndReadyMaster();
72
73
74 byte [] table = Bytes.toBytes("TestGlobalMemStoreSize");
75 byte [] family = Bytes.toBytes("family");
76 LOG.info("Creating table with " + regionNum + " regions");
77 HTable ht = TEST_UTIL.createTable(table, family);
78 int numRegions = TEST_UTIL.createMultiRegions(conf, ht, family,
79 regionNum);
80 assertEquals(regionNum,numRegions);
81 waitForAllRegionsAssigned();
82
83 for (HRegionServer server : getOnlineRegionServers()) {
84 long globalMemStoreSize = 0;
85 for (HRegionInfo regionInfo : ProtobufUtil.getOnlineRegions(server)) {
86 globalMemStoreSize +=
87 server.getFromOnlineRegions(regionInfo.getEncodedName()).
88 getMemstoreSize().get();
89 }
90 assertEquals(server.getRegionServerAccounting().getGlobalMemstoreSize(),
91 globalMemStoreSize);
92 }
93
94
95 int i = 0;
96 for (HRegionServer server : getOnlineRegionServers()) {
97 LOG.info("Starting flushes on " + server.getServerName() +
98 ", size=" + server.getRegionServerAccounting().getGlobalMemstoreSize());
99
100 for (HRegionInfo regionInfo : ProtobufUtil.getOnlineRegions(server)) {
101 HRegion r = server.getFromOnlineRegions(regionInfo.getEncodedName());
102 flush(r, server);
103 }
104 LOG.info("Post flush on " + server.getServerName());
105 long now = System.currentTimeMillis();
106 long timeout = now + 1000;
107 while(server.getRegionServerAccounting().getGlobalMemstoreSize() != 0 &&
108 timeout < System.currentTimeMillis()) {
109 Threads.sleep(10);
110 }
111 long size = server.getRegionServerAccounting().getGlobalMemstoreSize();
112 if (size > 0) {
113
114
115 for (HRegionInfo regionInfo : ProtobufUtil.getOnlineRegions(server)) {
116 HRegion r = server.getFromOnlineRegions(regionInfo.getEncodedName());
117 long l = r.getMemstoreSize().longValue();
118 if (l > 0) {
119
120
121 assertTrue(regionInfo.isMetaRegion());
122 LOG.info(r.toString() + " " + l + ", reflushing");
123 r.flushcache();
124 }
125 }
126 }
127 size = server.getRegionServerAccounting().getGlobalMemstoreSize();
128 assertEquals("Server=" + server.getServerName() + ", i=" + i++, 0, size);
129 }
130
131 ht.close();
132 TEST_UTIL.shutdownMiniCluster();
133 }
134
135
136
137
138
139
140
141 private void flush(final HRegion r, final HRegionServer server)
142 throws IOException {
143 LOG.info("Flush " + r.toString() + " on " + server.getServerName() +
144 ", " + r.flushcache() + ", size=" +
145 server.getRegionServerAccounting().getGlobalMemstoreSize());
146 }
147
148
149 private int getRegionCount() throws IOException {
150 int total = 0;
151 for (HRegionServer server : getOnlineRegionServers()) {
152 total += ProtobufUtil.getOnlineRegions(server).size();
153 }
154 return total;
155 }
156
157 private List<HRegionServer> getOnlineRegionServers() {
158 List<HRegionServer> list = new ArrayList<HRegionServer>();
159 for (JVMClusterUtil.RegionServerThread rst :
160 cluster.getRegionServerThreads()) {
161 if (rst.getRegionServer().isOnline()) {
162 list.add(rst.getRegionServer());
163 }
164 }
165 return list;
166 }
167
168
169
170
171 private void waitForAllRegionsAssigned() throws IOException {
172 while (getRegionCount() < totalRegionNum) {
173 LOG.debug("Waiting for there to be "+totalRegionNum+" regions, but there are " + getRegionCount() + " right now.");
174 try {
175 Thread.sleep(100);
176 } catch (InterruptedException e) {}
177 }
178 }
179
180 }