1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.List;
24 import java.util.NoSuchElementException;
25 import java.util.Random;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.hbase.classification.InterfaceAudience;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.TableName;
34 import org.apache.hadoop.hbase.HConstants;
35 import org.apache.hadoop.hbase.HRegionInfo;
36 import org.apache.hadoop.hbase.HTableDescriptor;
37 import org.apache.hadoop.hbase.RemoteExceptionHandler;
38 import org.apache.hadoop.hbase.TableNotDisabledException;
39 import org.apache.hadoop.hbase.catalog.MetaEditor;
40 import org.apache.hadoop.hbase.client.Delete;
41 import org.apache.hadoop.hbase.client.HBaseAdmin;
42 import org.apache.hadoop.hbase.client.HConnectable;
43 import org.apache.hadoop.hbase.client.HConnection;
44 import org.apache.hadoop.hbase.client.HConnectionManager;
45 import org.apache.hadoop.hbase.client.HTable;
46 import org.apache.hadoop.hbase.client.Result;
47 import org.apache.hadoop.hbase.client.ResultScanner;
48 import org.apache.hadoop.hbase.regionserver.HRegion;
49 import org.apache.hadoop.hbase.regionserver.wal.HLog;
50 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
51
52
53
54
55
56 @InterfaceAudience.Private
57 class HMerge {
58
59 static final Log LOG = LogFactory.getLog(HMerge.class);
60 static final Random rand = new Random();
61
62
63
64
65 private HMerge() {
66 super();
67 }
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82 public static void merge(Configuration conf, FileSystem fs,
83 final TableName tableName)
84 throws IOException {
85 merge(conf, fs, tableName, true);
86 }
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103 public static void merge(Configuration conf, FileSystem fs,
104 final TableName tableName, final boolean testMasterRunning)
105 throws IOException {
106 boolean masterIsRunning = false;
107 if (testMasterRunning) {
108 masterIsRunning = HConnectionManager
109 .execute(new HConnectable<Boolean>(conf) {
110 @Override
111 public Boolean connect(HConnection connection) throws IOException {
112 return connection.isMasterRunning();
113 }
114 });
115 }
116 if (tableName.equals(TableName.META_TABLE_NAME)) {
117 if (masterIsRunning) {
118 throw new IllegalStateException(
119 "Can not compact hbase:meta table if instance is on-line");
120 }
121
122 } else {
123 if(!masterIsRunning) {
124 throw new IllegalStateException(
125 "HBase instance must be running to merge a normal table");
126 }
127 HBaseAdmin admin = new HBaseAdmin(conf);
128 if (!admin.isTableDisabled(tableName)) {
129 throw new TableNotDisabledException(tableName);
130 }
131 new OnlineMerger(conf, fs, tableName).process();
132 }
133 }
134
135 private static abstract class Merger {
136 protected final Configuration conf;
137 protected final FileSystem fs;
138 protected final Path rootDir;
139 protected final HTableDescriptor htd;
140 protected final HLog hlog;
141 private final long maxFilesize;
142
143
144 protected Merger(Configuration conf, FileSystem fs, final TableName tableName)
145 throws IOException {
146 this.conf = conf;
147 this.fs = fs;
148 this.maxFilesize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
149 HConstants.DEFAULT_MAX_FILE_SIZE);
150
151 this.rootDir = FSUtils.getRootDir(conf);
152 Path tabledir = FSUtils.getTableDir(this.rootDir, tableName);
153 this.htd = FSTableDescriptors.getTableDescriptorFromFs(this.fs, tabledir);
154 String logname = "merge_" + System.currentTimeMillis() + HConstants.HREGION_LOGDIR_NAME;
155
156 this.hlog = HLogFactory.createHLog(fs, tabledir, logname, conf);
157 }
158
159 void process() throws IOException {
160 try {
161 for (HRegionInfo[] regionsToMerge = next();
162 regionsToMerge != null;
163 regionsToMerge = next()) {
164 if (!merge(regionsToMerge)) {
165 return;
166 }
167 }
168 } finally {
169 try {
170 hlog.closeAndDelete();
171
172 } catch(IOException e) {
173 LOG.error(e);
174 }
175 }
176 }
177
178 protected boolean merge(final HRegionInfo[] info) throws IOException {
179 if (info.length < 2) {
180 LOG.info("only one region - nothing to merge");
181 return false;
182 }
183
184 HRegion currentRegion = null;
185 long currentSize = 0;
186 HRegion nextRegion = null;
187 long nextSize = 0;
188 for (int i = 0; i < info.length - 1; i++) {
189 if (currentRegion == null) {
190 currentRegion = HRegion.openHRegion(conf, fs, this.rootDir, info[i], this.htd, hlog);
191 currentSize = currentRegion.getLargestHStoreSize();
192 }
193 nextRegion = HRegion.openHRegion(conf, fs, this.rootDir, info[i + 1], this.htd, hlog);
194 nextSize = nextRegion.getLargestHStoreSize();
195
196 if ((currentSize + nextSize) <= (maxFilesize / 2)) {
197
198
199 LOG.info("Merging regions " + currentRegion.getRegionNameAsString() +
200 " and " + nextRegion.getRegionNameAsString());
201 HRegion mergedRegion =
202 HRegion.mergeAdjacent(currentRegion, nextRegion);
203 updateMeta(currentRegion.getRegionName(), nextRegion.getRegionName(),
204 mergedRegion);
205 break;
206 }
207 LOG.info("not merging regions " + Bytes.toStringBinary(currentRegion.getRegionName())
208 + " and " + Bytes.toStringBinary(nextRegion.getRegionName()));
209 currentRegion.close();
210 currentRegion = nextRegion;
211 currentSize = nextSize;
212 }
213 if(currentRegion != null) {
214 currentRegion.close();
215 }
216 return true;
217 }
218
219 protected abstract HRegionInfo[] next() throws IOException;
220
221 protected abstract void updateMeta(final byte [] oldRegion1,
222 final byte [] oldRegion2, HRegion newRegion)
223 throws IOException;
224
225 }
226
227
228 private static class OnlineMerger extends Merger {
229 private final TableName tableName;
230 private final HTable table;
231 private final ResultScanner metaScanner;
232 private HRegionInfo latestRegion;
233
234 OnlineMerger(Configuration conf, FileSystem fs,
235 final TableName tableName)
236 throws IOException {
237 super(conf, fs, tableName);
238 this.tableName = tableName;
239 this.table = new HTable(conf, TableName.META_TABLE_NAME);
240 this.metaScanner = table.getScanner(HConstants.CATALOG_FAMILY,
241 HConstants.REGIONINFO_QUALIFIER);
242 this.latestRegion = null;
243 }
244
245 private HRegionInfo nextRegion() throws IOException {
246 try {
247 Result results = getMetaRow();
248 if (results == null) {
249 return null;
250 }
251 HRegionInfo region = HRegionInfo.getHRegionInfo(results);
252 if (region == null) {
253 throw new NoSuchElementException("meta region entry missing " +
254 Bytes.toString(HConstants.CATALOG_FAMILY) + ":" +
255 Bytes.toString(HConstants.REGIONINFO_QUALIFIER));
256 }
257 if (!region.getTable().equals(this.tableName)) {
258 return null;
259 }
260 return region;
261 } catch (IOException e) {
262 e = RemoteExceptionHandler.checkIOException(e);
263 LOG.error("meta scanner error", e);
264 metaScanner.close();
265 throw e;
266 }
267 }
268
269
270
271
272
273
274 private Result getMetaRow() throws IOException {
275 Result currentRow = metaScanner.next();
276 boolean foundResult = false;
277 while (currentRow != null) {
278 LOG.info("Row: <" + Bytes.toStringBinary(currentRow.getRow()) + ">");
279 byte[] regionInfoValue = currentRow.getValue(HConstants.CATALOG_FAMILY,
280 HConstants.REGIONINFO_QUALIFIER);
281 if (regionInfoValue == null || regionInfoValue.length == 0) {
282 currentRow = metaScanner.next();
283 continue;
284 }
285 HRegionInfo region = HRegionInfo.getHRegionInfo(currentRow);
286 if (!region.getTable().equals(this.tableName)) {
287 currentRow = metaScanner.next();
288 continue;
289 }
290 foundResult = true;
291 break;
292 }
293 return foundResult ? currentRow : null;
294 }
295
296 @Override
297 protected HRegionInfo[] next() throws IOException {
298 List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
299 if(latestRegion == null) {
300 latestRegion = nextRegion();
301 }
302 if(latestRegion != null) {
303 regions.add(latestRegion);
304 }
305 latestRegion = nextRegion();
306 if(latestRegion != null) {
307 regions.add(latestRegion);
308 }
309 return regions.toArray(new HRegionInfo[regions.size()]);
310 }
311
312 @Override
313 protected void updateMeta(final byte [] oldRegion1,
314 final byte [] oldRegion2,
315 HRegion newRegion)
316 throws IOException {
317 byte[][] regionsToDelete = {oldRegion1, oldRegion2};
318 for (int r = 0; r < regionsToDelete.length; r++) {
319 if(Bytes.equals(regionsToDelete[r], latestRegion.getRegionName())) {
320 latestRegion = null;
321 }
322 Delete delete = new Delete(regionsToDelete[r]);
323 table.delete(delete);
324 if(LOG.isDebugEnabled()) {
325 LOG.debug("updated columns in row: " + Bytes.toStringBinary(regionsToDelete[r]));
326 }
327 }
328 newRegion.getRegionInfo().setOffline(true);
329
330 MetaEditor.addRegionToMeta(table, newRegion.getRegionInfo());
331
332 if(LOG.isDebugEnabled()) {
333 LOG.debug("updated columns in row: "
334 + Bytes.toStringBinary(newRegion.getRegionName()));
335 }
336 }
337 }
338 }