View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.List;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.fs.FileSystem;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.Cell;
30  import org.apache.hadoop.hbase.CellUtil;
31  import org.apache.hadoop.hbase.HBaseTestCase;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.KeyValue;
36  import org.apache.hadoop.hbase.testclassification.MediumTests;
37  import org.apache.hadoop.hbase.TableName;
38  import org.apache.hadoop.hbase.catalog.MetaEditor;
39  import org.apache.hadoop.hbase.client.Delete;
40  import org.apache.hadoop.hbase.client.Durability;
41  import org.apache.hadoop.hbase.client.Put;
42  import org.apache.hadoop.hbase.client.Result;
43  import org.apache.hadoop.hbase.client.Scan;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.junit.experimental.categories.Category;
46  
47  /**
48   * TestGet is a medley of tests of get all done up as a single test.
49   * This class
50   */
51  @Category(MediumTests.class)
52  public class TestGetClosestAtOrBefore extends HBaseTestCase {
53    private static final Log LOG = LogFactory.getLog(TestGetClosestAtOrBefore.class);
54  
55    private static final byte[] T00 = Bytes.toBytes("000");
56    private static final byte[] T10 = Bytes.toBytes("010");
57    private static final byte[] T11 = Bytes.toBytes("011");
58    private static final byte[] T12 = Bytes.toBytes("012");
59    private static final byte[] T20 = Bytes.toBytes("020");
60    private static final byte[] T30 = Bytes.toBytes("030");
61    private static final byte[] T31 = Bytes.toBytes("031");
62    private static final byte[] T35 = Bytes.toBytes("035");
63    private static final byte[] T40 = Bytes.toBytes("040");
64  
65  
66  
67    public void testUsingMetaAndBinary() throws IOException {
68      FileSystem filesystem = FileSystem.get(conf);
69      Path rootdir = testDir;
70      // Up flush size else we bind up when we use default catalog flush of 16k.
71      fsTableDescriptors.get(TableName.META_TABLE_NAME).setMemStoreFlushSize(64 * 1024 * 1024);
72      HRegion mr = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO,
73        rootdir, this.conf, fsTableDescriptors.get(TableName.META_TABLE_NAME));
74      try {
75      // Write rows for three tables 'A', 'B', and 'C'.
76      for (char c = 'A'; c < 'D'; c++) {
77        HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("" + c));
78        final int last = 128;
79        final int interval = 2;
80        for (int i = 0; i <= last; i += interval) {
81          HRegionInfo hri = new HRegionInfo(htd.getTableName(),
82            i == 0? HConstants.EMPTY_BYTE_ARRAY: Bytes.toBytes((byte)i),
83            i == last? HConstants.EMPTY_BYTE_ARRAY: Bytes.toBytes((byte)i + interval));
84  
85          Put put = MetaEditor.makePutFromRegionInfo(hri);
86          put.setDurability(Durability.SKIP_WAL);
87          mr.put(put);
88        }
89      }
90      InternalScanner s = mr.getScanner(new Scan());
91      try {
92        List<Cell> keys = new ArrayList<Cell>();
93        while(s.next(keys)) {
94          LOG.info(keys);
95          keys.clear();
96        }
97      } finally {
98        s.close();
99      }
100     findRow(mr, 'C', 44, 44);
101     findRow(mr, 'C', 45, 44);
102     findRow(mr, 'C', 46, 46);
103     findRow(mr, 'C', 43, 42);
104     mr.flushcache();
105     findRow(mr, 'C', 44, 44);
106     findRow(mr, 'C', 45, 44);
107     findRow(mr, 'C', 46, 46);
108     findRow(mr, 'C', 43, 42);
109     // Now delete 'C' and make sure I don't get entries from 'B'.
110     byte [] firstRowInC = HRegionInfo.createRegionName(
111         TableName.valueOf("" + 'C'),
112         HConstants.EMPTY_BYTE_ARRAY, HConstants.ZEROES, false);
113     Scan scan = new Scan(firstRowInC);
114     s = mr.getScanner(scan);
115     try {
116       List<Cell> keys = new ArrayList<Cell>();
117       while (s.next(keys)) {
118         mr.delete(new Delete(CellUtil.cloneRow(keys.get(0))));
119         keys.clear();
120       }
121     } finally {
122       s.close();
123     }
124     // Assert we get null back (pass -1).
125     findRow(mr, 'C', 44, -1);
126     findRow(mr, 'C', 45, -1);
127     findRow(mr, 'C', 46, -1);
128     findRow(mr, 'C', 43, -1);
129     mr.flushcache();
130     findRow(mr, 'C', 44, -1);
131     findRow(mr, 'C', 45, -1);
132     findRow(mr, 'C', 46, -1);
133     findRow(mr, 'C', 43, -1);
134     } finally {
135       if (mr != null) {
136         try {
137           mr.close();
138         } catch (Exception e) {
139           e.printStackTrace();
140         }
141         mr.getLog().closeAndDelete();
142       }
143     }
144   }
145 
146   /*
147    * @param mr
148    * @param table
149    * @param rowToFind
150    * @param answer Pass -1 if we're not to find anything.
151    * @return Row found.
152    * @throws IOException
153    */
154   private byte [] findRow(final HRegion mr, final char table,
155     final int rowToFind, final int answer)
156   throws IOException {
157     TableName tableb = TableName.valueOf("" + table);
158     // Find the row.
159     byte [] tofindBytes = Bytes.toBytes((short)rowToFind);
160     byte [] metaKey = HRegionInfo.createRegionName(
161         tableb, tofindBytes,
162       HConstants.NINES, false);
163     LOG.info("find=" + new String(metaKey));
164     Result r = mr.getClosestRowBefore(metaKey);
165     if (answer == -1) {
166       assertNull(r);
167       return null;
168     }
169     assertTrue(Bytes.compareTo(Bytes.toBytes((short)answer),
170       extractRowFromMetaRow(r.getRow())) == 0);
171     return r.getRow();
172   }
173 
174   private byte [] extractRowFromMetaRow(final byte [] b) {
175     int firstDelimiter = KeyValue.getDelimiter(b, 0, b.length,
176       HConstants.DELIMITER);
177     int lastDelimiter = KeyValue.getDelimiterInReverse(b, 0, b.length,
178       HConstants.DELIMITER);
179     int length = lastDelimiter - firstDelimiter - 1;
180     byte [] row = new byte[length];
181     System.arraycopy(b, firstDelimiter + 1, row, 0, length);
182     return row;
183   }
184 
185   /**
186    * Test file of multiple deletes and with deletes as final key.
187    * @see <a href="https://issues.apache.org/jira/browse/HBASE-751">HBASE-751</a>
188    */
189   public void testGetClosestRowBefore3() throws IOException{
190     HRegion region = null;
191     byte [] c0 = COLUMNS[0];
192     byte [] c1 = COLUMNS[1];
193     try {
194       HTableDescriptor htd = createTableDescriptor(getName());
195       region = createNewHRegion(htd, null, null);
196 
197       Put p = new Put(T00);
198       p.add(c0, c0, T00);
199       region.put(p);
200 
201       p = new Put(T10);
202       p.add(c0, c0, T10);
203       region.put(p);
204 
205       p = new Put(T20);
206       p.add(c0, c0, T20);
207       region.put(p);
208 
209       Result r = region.getClosestRowBefore(T20, c0);
210       assertTrue(Bytes.equals(T20, r.getRow()));
211 
212       Delete d = new Delete(T20);
213       d.deleteColumn(c0, c0);
214       region.delete(d);
215 
216       r = region.getClosestRowBefore(T20, c0);
217       assertTrue(Bytes.equals(T10, r.getRow()));
218 
219       p = new Put(T30);
220       p.add(c0, c0, T30);
221       region.put(p);
222 
223       r = region.getClosestRowBefore(T30, c0);
224       assertTrue(Bytes.equals(T30, r.getRow()));
225 
226       d = new Delete(T30);
227       d.deleteColumn(c0, c0);
228       region.delete(d);
229 
230       r = region.getClosestRowBefore(T30, c0);
231       assertTrue(Bytes.equals(T10, r.getRow()));
232       r = region.getClosestRowBefore(T31, c0);
233       assertTrue(Bytes.equals(T10, r.getRow()));
234 
235       region.flushcache();
236 
237       // try finding "010" after flush
238       r = region.getClosestRowBefore(T30, c0);
239       assertTrue(Bytes.equals(T10, r.getRow()));
240       r = region.getClosestRowBefore(T31, c0);
241       assertTrue(Bytes.equals(T10, r.getRow()));
242 
243       // Put into a different column family.  Should make it so I still get t10
244       p = new Put(T20);
245       p.add(c1, c1, T20);
246       region.put(p);
247 
248       r = region.getClosestRowBefore(T30, c0);
249       assertTrue(Bytes.equals(T10, r.getRow()));
250       r = region.getClosestRowBefore(T31, c0);
251       assertTrue(Bytes.equals(T10, r.getRow()));
252 
253       region.flushcache();
254 
255       r = region.getClosestRowBefore(T30, c0);
256       assertTrue(Bytes.equals(T10, r.getRow()));
257       r = region.getClosestRowBefore(T31, c0);
258       assertTrue(Bytes.equals(T10, r.getRow()));
259 
260       // Now try combo of memcache and mapfiles.  Delete the t20 COLUMS[1]
261       // in memory; make sure we get back t10 again.
262       d = new Delete(T20);
263       d.deleteColumn(c1, c1);
264       region.delete(d);
265       r = region.getClosestRowBefore(T30, c0);
266       assertTrue(Bytes.equals(T10, r.getRow()));
267 
268       // Ask for a value off the end of the file.  Should return t10.
269       r = region.getClosestRowBefore(T31, c0);
270       assertTrue(Bytes.equals(T10, r.getRow()));
271       region.flushcache();
272       r = region.getClosestRowBefore(T31, c0);
273       assertTrue(Bytes.equals(T10, r.getRow()));
274 
275       // Ok.  Let the candidate come out of hfile but have delete of
276       // the candidate be in memory.
277       p = new Put(T11);
278       p.add(c0, c0, T11);
279       region.put(p);
280       d = new Delete(T10);
281       d.deleteColumn(c1, c1);
282       r = region.getClosestRowBefore(T12, c0);
283       assertTrue(Bytes.equals(T11, r.getRow()));
284     } finally {
285       if (region != null) {
286         try {
287           region.close();
288         } catch (Exception e) {
289           e.printStackTrace();
290         }
291         region.getLog().closeAndDelete();
292       }
293     }
294   }
295 
296   /** For HBASE-694 */
297   public void testGetClosestRowBefore2() throws IOException{
298     HRegion region = null;
299     byte [] c0 = COLUMNS[0];
300     try {
301       HTableDescriptor htd = createTableDescriptor(getName());
302       region = createNewHRegion(htd, null, null);
303 
304       Put p = new Put(T10);
305       p.add(c0, c0, T10);
306       region.put(p);
307 
308       p = new Put(T30);
309       p.add(c0, c0, T30);
310       region.put(p);
311 
312       p = new Put(T40);
313       p.add(c0, c0, T40);
314       region.put(p);
315 
316       // try finding "035"
317       Result r = region.getClosestRowBefore(T35, c0);
318       assertTrue(Bytes.equals(T30, r.getRow()));
319 
320       region.flushcache();
321 
322       // try finding "035"
323       r = region.getClosestRowBefore(T35, c0);
324       assertTrue(Bytes.equals(T30, r.getRow()));
325 
326       p = new Put(T20);
327       p.add(c0, c0, T20);
328       region.put(p);
329 
330       // try finding "035"
331       r = region.getClosestRowBefore(T35, c0);
332       assertTrue(Bytes.equals(T30, r.getRow()));
333 
334       region.flushcache();
335 
336       // try finding "035"
337       r = region.getClosestRowBefore(T35, c0);
338       assertTrue(Bytes.equals(T30, r.getRow()));
339     } finally {
340       if (region != null) {
341         try {
342           region.close();
343         } catch (Exception e) {
344           e.printStackTrace();
345         }
346         region.getLog().closeAndDelete();
347       }
348     }
349   }
350 
351 }
352