View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.IOException;
26  import java.util.ArrayList;
27  import java.util.List;
28  import java.util.TreeMap;
29  
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.TableName;
32  import org.apache.hadoop.hbase.HBaseConfiguration;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.HRegionInfo;
35  import org.apache.hadoop.hbase.HTableDescriptor;
36  import org.apache.hadoop.hbase.testclassification.SmallTests;
37  import org.apache.hadoop.hbase.util.Bytes;
38  import org.junit.Before;
39  import org.junit.Test;
40  import org.junit.experimental.categories.Category;
41  import org.mockito.Mockito;
42  
43  @Category(SmallTests.class)
44  public class TestRegionSplitPolicy {
45  
46    private Configuration conf;
47    private HTableDescriptor htd;
48    private HRegion mockRegion;
49    private TreeMap<byte[], HStore> stores;
50    private static final TableName TABLENAME = TableName.valueOf("t");
51  
52    @Before
53    public void setupMocks() {
54      conf = HBaseConfiguration.create();
55      HRegionInfo hri = new HRegionInfo(TABLENAME);
56      htd = new HTableDescriptor(TABLENAME);
57      mockRegion = Mockito.mock(HRegion.class);
58      Mockito.doReturn(htd).when(mockRegion).getTableDesc();
59      Mockito.doReturn(hri).when(mockRegion).getRegionInfo();
60  
61      stores = new TreeMap<byte[], HStore>(Bytes.BYTES_COMPARATOR);
62      Mockito.doReturn(stores).when(mockRegion).getStores();
63    }
64  
65    @Test
66    public void testIncreasingToUpperBoundRegionSplitPolicy() throws IOException {
67      // Configure IncreasingToUpperBoundRegionSplitPolicy as our split policy
68      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
69        IncreasingToUpperBoundRegionSplitPolicy.class.getName());
70      // Now make it so the mock region has a RegionServerService that will
71      // return 'online regions'.
72      RegionServerServices rss = Mockito.mock(RegionServerServices.class);
73      final List<HRegion> regions = new ArrayList<HRegion>();
74      Mockito.when(rss.getOnlineRegions(TABLENAME)).thenReturn(regions);
75      Mockito.when(mockRegion.getRegionServerServices()).thenReturn(rss);
76      // Set max size for this 'table'.
77      long maxSplitSize = 1024L;
78      htd.setMaxFileSize(maxSplitSize);
79      // Set flush size to 1/8.  IncreasingToUpperBoundRegionSplitPolicy
80      // grows by the cube of the number of regions times flushsize each time.
81      long flushSize = maxSplitSize/8;
82      conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSize);
83      htd.setMemStoreFlushSize(flushSize);
84      // If RegionServerService with no regions in it -- 'online regions' == 0 --
85      // then IncreasingToUpperBoundRegionSplitPolicy should act like a
86      // ConstantSizePolicy
87      IncreasingToUpperBoundRegionSplitPolicy policy =
88        (IncreasingToUpperBoundRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
89      doConstantSizePolicyTests(policy);
90  
91      // Add a store in excess of split size.  Because there are "no regions"
92      // on this server -- rss.getOnlineRegions is 0 -- then we should split
93      // like a constantsizeregionsplitpolicy would
94      HStore mockStore = Mockito.mock(HStore.class);
95      Mockito.doReturn(2000L).when(mockStore).getSize();
96      Mockito.doReturn(true).when(mockStore).canSplit();
97      stores.put(new byte[]{1}, mockStore);
98      // It should split
99      assertTrue(policy.shouldSplit());
100 
101     // Now test that we increase our split size as online regions for a table
102     // grows. With one region, split size should be flushsize.
103     regions.add(mockRegion);
104     Mockito.doReturn(flushSize).when(mockStore).getSize();
105     // Should not split since store is flush size.
106     assertFalse(policy.shouldSplit());
107     // Set size of store to be > 2*flush size and we should split
108     Mockito.doReturn(flushSize*2 + 1).when(mockStore).getSize();
109     assertTrue(policy.shouldSplit());
110     // Add another region to the 'online regions' on this server and we should
111     // now be no longer be splittable since split size has gone up.
112     regions.add(mockRegion);
113     assertFalse(policy.shouldSplit());
114     // Quadruple (2 squared) the store size and make sure its just over; verify it'll split
115     Mockito.doReturn((flushSize * 2 * 2 * 2) + 1).when(mockStore).getSize();
116     assertTrue(policy.shouldSplit());
117 
118     // Finally assert that even if loads of regions, we'll split at max size
119     assertEquals(maxSplitSize, policy.getSizeToCheck(1000));
120     // Assert same is true if count of regions is zero.
121     assertEquals(maxSplitSize, policy.getSizeToCheck(0));
122   }
123 
124   @Test
125   public void testCreateDefault() throws IOException {
126     conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1234L);
127 
128     // Using a default HTD, should pick up the file size from
129     // configuration.
130     ConstantSizeRegionSplitPolicy policy =
131         (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(
132             mockRegion, conf);
133     assertEquals(1234L, policy.getDesiredMaxFileSize());
134 
135     // If specified in HTD, should use that
136     htd.setMaxFileSize(9999L);
137     policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(
138         mockRegion, conf);
139     assertEquals(9999L, policy.getDesiredMaxFileSize());
140   }
141 
142   /**
143    * Test setting up a customized split policy
144    */
145   @Test
146   public void testCustomPolicy() throws IOException {
147     HTableDescriptor myHtd = new HTableDescriptor();
148     myHtd.setValue(HTableDescriptor.SPLIT_POLICY,
149         KeyPrefixRegionSplitPolicy.class.getName());
150     myHtd.setValue(KeyPrefixRegionSplitPolicy.PREFIX_LENGTH_KEY, String.valueOf(2));
151 
152     HRegion myMockRegion = Mockito.mock(HRegion.class);
153     Mockito.doReturn(myHtd).when(myMockRegion).getTableDesc();
154     Mockito.doReturn(stores).when(myMockRegion).getStores();
155 
156     HStore mockStore = Mockito.mock(HStore.class);
157     Mockito.doReturn(2000L).when(mockStore).getSize();
158     Mockito.doReturn(true).when(mockStore).canSplit();
159     Mockito.doReturn(Bytes.toBytes("abcd")).when(mockStore).getSplitPoint();
160     stores.put(new byte[] { 1 }, mockStore);
161 
162     KeyPrefixRegionSplitPolicy policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy
163         .create(myMockRegion, conf);
164 
165     assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
166 
167     Mockito.doReturn(true).when(myMockRegion).shouldForceSplit();
168     Mockito.doReturn(Bytes.toBytes("efgh")).when(myMockRegion)
169         .getExplicitSplitPoint();
170 
171     policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy
172         .create(myMockRegion, conf);
173 
174     assertEquals("ef", Bytes.toString(policy.getSplitPoint()));
175   }
176 
177   @Test
178   public void testConstantSizePolicy() throws IOException {
179     htd.setMaxFileSize(1024L);
180     ConstantSizeRegionSplitPolicy policy =
181       (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
182     doConstantSizePolicyTests(policy);
183   }
184 
185   /**
186    * Run through tests for a ConstantSizeRegionSplitPolicy
187    * @param policy
188    */
189   private void doConstantSizePolicyTests(final ConstantSizeRegionSplitPolicy policy) {
190     // For no stores, should not split
191     assertFalse(policy.shouldSplit());
192 
193     // Add a store above the requisite size. Should split.
194     HStore mockStore = Mockito.mock(HStore.class);
195     Mockito.doReturn(2000L).when(mockStore).getSize();
196     Mockito.doReturn(true).when(mockStore).canSplit();
197     stores.put(new byte[]{1}, mockStore);
198 
199     assertTrue(policy.shouldSplit());
200 
201     // Act as if there's a reference file or some other reason it can't split.
202     // This should prevent splitting even though it's big enough.
203     Mockito.doReturn(false).when(mockStore).canSplit();
204     assertFalse(policy.shouldSplit());
205 
206     // Reset splittability after above
207     Mockito.doReturn(true).when(mockStore).canSplit();
208 
209     // Set to a small size but turn on forceSplit. Should result in a split.
210     Mockito.doReturn(true).when(mockRegion).shouldForceSplit();
211     Mockito.doReturn(100L).when(mockStore).getSize();
212     assertTrue(policy.shouldSplit());
213 
214     // Turn off forceSplit, should not split
215     Mockito.doReturn(false).when(mockRegion).shouldForceSplit();
216     assertFalse(policy.shouldSplit());
217 
218     // Clear families we added above
219     stores.clear();
220   }
221 
222   @Test
223   public void testGetSplitPoint() throws IOException {
224     ConstantSizeRegionSplitPolicy policy =
225       (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
226 
227     // For no stores, should not split
228     assertFalse(policy.shouldSplit());
229     assertNull(policy.getSplitPoint());
230 
231     // Add a store above the requisite size. Should split.
232     HStore mockStore = Mockito.mock(HStore.class);
233     Mockito.doReturn(2000L).when(mockStore).getSize();
234     Mockito.doReturn(true).when(mockStore).canSplit();
235     Mockito.doReturn(Bytes.toBytes("store 1 split"))
236       .when(mockStore).getSplitPoint();
237     stores.put(new byte[]{1}, mockStore);
238 
239     assertEquals("store 1 split",
240         Bytes.toString(policy.getSplitPoint()));
241 
242     // Add a bigger store. The split point should come from that one
243     HStore mockStore2 = Mockito.mock(HStore.class);
244     Mockito.doReturn(4000L).when(mockStore2).getSize();
245     Mockito.doReturn(true).when(mockStore2).canSplit();
246     Mockito.doReturn(Bytes.toBytes("store 2 split"))
247       .when(mockStore2).getSplitPoint();
248     stores.put(new byte[]{2}, mockStore2);
249 
250     assertEquals("store 2 split",
251         Bytes.toString(policy.getSplitPoint()));
252   }
253 
254   @Test
255   public void testDelimitedKeyPrefixRegionSplitPolicy() throws IOException {
256     HTableDescriptor myHtd = new HTableDescriptor();
257     myHtd.setValue(HTableDescriptor.SPLIT_POLICY,
258         DelimitedKeyPrefixRegionSplitPolicy.class.getName());
259     myHtd.setValue(DelimitedKeyPrefixRegionSplitPolicy.DELIMITER_KEY, ",");
260 
261     HRegion myMockRegion = Mockito.mock(HRegion.class);
262     Mockito.doReturn(myHtd).when(myMockRegion).getTableDesc();
263     Mockito.doReturn(stores).when(myMockRegion).getStores();
264 
265     HStore mockStore = Mockito.mock(HStore.class);
266     Mockito.doReturn(2000L).when(mockStore).getSize();
267     Mockito.doReturn(true).when(mockStore).canSplit();
268     Mockito.doReturn(Bytes.toBytes("ab,cd")).when(mockStore).getSplitPoint();
269     stores.put(new byte[] { 1 }, mockStore);
270 
271     DelimitedKeyPrefixRegionSplitPolicy policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy
272         .create(myMockRegion, conf);
273 
274     assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
275 
276     Mockito.doReturn(true).when(myMockRegion).shouldForceSplit();
277     Mockito.doReturn(Bytes.toBytes("efg,h")).when(myMockRegion)
278         .getExplicitSplitPoint();
279 
280     policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy
281         .create(myMockRegion, conf);
282 
283     assertEquals("efg", Bytes.toString(policy.getSplitPoint()));
284 
285     Mockito.doReturn(Bytes.toBytes("ijk")).when(myMockRegion)
286     .getExplicitSplitPoint();
287     assertEquals("ijk", Bytes.toString(policy.getSplitPoint()));
288   }
289 
290 }