1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.catalog;
19
20 import java.io.IOException;
21 import java.io.InterruptedIOException;
22 import java.net.ConnectException;
23 import java.util.ArrayList;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.hbase.classification.InterfaceAudience;
29 import org.apache.hadoop.hbase.DoNotRetryIOException;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.HRegionInfo;
32 import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
33 import org.apache.hadoop.hbase.ServerName;
34 import org.apache.hadoop.hbase.client.Delete;
35 import org.apache.hadoop.hbase.client.HTable;
36 import org.apache.hadoop.hbase.client.Mutation;
37 import org.apache.hadoop.hbase.client.Put;
38 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
39 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
40 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
41 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
42 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
45 import org.apache.hadoop.hbase.util.Threads;
46
47 import com.google.protobuf.ServiceException;
48
49
50
51
52
53
54 @InterfaceAudience.Private
55 public class MetaEditor {
56
57
58
59 private static final Log LOG = LogFactory.getLog(MetaEditor.class);
60
61
62
63
64 public static Put makePutFromRegionInfo(HRegionInfo regionInfo)
65 throws IOException {
66 Put put = new Put(regionInfo.getRegionName());
67 addRegionInfo(put, regionInfo);
68 return put;
69 }
70
71
72
73
74
75 public static Delete makeDeleteFromRegionInfo(HRegionInfo regionInfo) {
76 if (regionInfo == null) {
77 throw new IllegalArgumentException("Can't make a delete for null region");
78 }
79 Delete delete = new Delete(regionInfo.getRegionName());
80 return delete;
81 }
82
83
84
85
86 public static Put addDaughtersToPut(Put put, HRegionInfo splitA, HRegionInfo splitB) {
87 if (splitA != null) {
88 put.addImmutable(
89 HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER, splitA.toByteArray());
90 }
91 if (splitB != null) {
92 put.addImmutable(
93 HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER, splitB.toByteArray());
94 }
95 return put;
96 }
97
98
99
100
101
102
103
104 static void putToMetaTable(final CatalogTracker ct, final Put p)
105 throws IOException {
106 put(MetaReader.getMetaHTable(ct), p);
107 }
108
109
110
111
112
113
114
115 static void putToCatalogTable(final CatalogTracker ct, final Put p)
116 throws IOException {
117 put(MetaReader.getCatalogHTable(ct), p);
118 }
119
120
121
122
123
124
125 private static void put(final HTable t, final Put p) throws IOException {
126 try {
127 t.put(p);
128 } finally {
129 t.close();
130 }
131 }
132
133
134
135
136
137
138
139 public static void putsToMetaTable(final CatalogTracker ct, final List<Put> ps)
140 throws IOException {
141 HTable t = MetaReader.getMetaHTable(ct);
142 try {
143 t.put(ps);
144 } finally {
145 t.close();
146 }
147 }
148
149
150
151
152
153
154
155 static void deleteFromMetaTable(final CatalogTracker ct, final Delete d)
156 throws IOException {
157 List<Delete> dels = new ArrayList<Delete>(1);
158 dels.add(d);
159 deleteFromMetaTable(ct, dels);
160 }
161
162
163
164
165
166
167
168 public static void deleteFromMetaTable(final CatalogTracker ct, final List<Delete> deletes)
169 throws IOException {
170 HTable t = MetaReader.getMetaHTable(ct);
171 try {
172 t.delete(deletes);
173 } finally {
174 t.close();
175 }
176 }
177
178
179
180
181
182
183
184 public static void mutateMetaTable(final CatalogTracker ct, final List<Mutation> mutations)
185 throws IOException {
186 HTable t = MetaReader.getMetaHTable(ct);
187 try {
188 t.batch(mutations);
189 } catch (InterruptedException e) {
190 InterruptedIOException ie = new InterruptedIOException(e.getMessage());
191 ie.initCause(e);
192 throw ie;
193 } finally {
194 t.close();
195 }
196 }
197
198
199
200
201
202
203 public static void addRegionToMeta(CatalogTracker catalogTracker,
204 HRegionInfo regionInfo)
205 throws IOException {
206 putToMetaTable(catalogTracker, makePutFromRegionInfo(regionInfo));
207 LOG.info("Added " + regionInfo.getRegionNameAsString());
208 }
209
210
211
212
213
214
215
216
217 public static void addRegionToMeta(HTable meta, HRegionInfo regionInfo) throws IOException {
218 addRegionToMeta(meta, regionInfo, null, null);
219 }
220
221
222
223
224
225
226
227
228
229
230
231
232
233 public static void addRegionToMeta(HTable meta, HRegionInfo regionInfo,
234 HRegionInfo splitA, HRegionInfo splitB) throws IOException {
235 Put put = makePutFromRegionInfo(regionInfo);
236 addDaughtersToPut(put, splitA, splitB);
237 meta.put(put);
238 if (LOG.isDebugEnabled()) {
239 LOG.debug("Added " + regionInfo.getRegionNameAsString());
240 }
241 }
242
243
244
245
246
247
248
249
250
251
252
253
254
255 public static void addRegionToMeta(CatalogTracker catalogTracker, HRegionInfo regionInfo,
256 HRegionInfo splitA, HRegionInfo splitB) throws IOException {
257 HTable meta = MetaReader.getMetaHTable(catalogTracker);
258 try {
259 addRegionToMeta(meta, regionInfo, splitA, splitB);
260 } finally {
261 meta.close();
262 }
263 }
264
265
266
267
268
269
270
271 public static void addRegionsToMeta(CatalogTracker catalogTracker,
272 List<HRegionInfo> regionInfos)
273 throws IOException {
274 List<Put> puts = new ArrayList<Put>();
275 for (HRegionInfo regionInfo : regionInfos) {
276 puts.add(makePutFromRegionInfo(regionInfo));
277 }
278 putsToMetaTable(catalogTracker, puts);
279 LOG.info("Added " + puts.size());
280 }
281
282
283
284
285
286
287
288 public static void addDaughter(final CatalogTracker catalogTracker,
289 final HRegionInfo regionInfo, final ServerName sn, final long openSeqNum)
290 throws NotAllMetaRegionsOnlineException, IOException {
291 Put put = new Put(regionInfo.getRegionName());
292 addRegionInfo(put, regionInfo);
293 if (sn != null) {
294 addLocation(put, sn, openSeqNum);
295 }
296 putToMetaTable(catalogTracker, put);
297 LOG.info("Added daughter " + regionInfo.getEncodedName() +
298 (sn == null? ", serverName=null": ", serverName=" + sn.toString()));
299 }
300
301
302
303
304
305
306
307
308
309
310
311
312 public static void mergeRegions(final CatalogTracker catalogTracker,
313 HRegionInfo mergedRegion, HRegionInfo regionA, HRegionInfo regionB,
314 ServerName sn) throws IOException {
315 HTable meta = MetaReader.getMetaHTable(catalogTracker);
316 try {
317 HRegionInfo copyOfMerged = new HRegionInfo(mergedRegion);
318
319
320 Put putOfMerged = makePutFromRegionInfo(copyOfMerged);
321 putOfMerged.addImmutable(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER,
322 regionA.toByteArray());
323 putOfMerged.addImmutable(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER,
324 regionB.toByteArray());
325
326
327 Delete deleteA = makeDeleteFromRegionInfo(regionA);
328 Delete deleteB = makeDeleteFromRegionInfo(regionB);
329
330
331 addLocation(putOfMerged, sn, 1);
332
333 byte[] tableRow = Bytes.toBytes(mergedRegion.getRegionNameAsString()
334 + HConstants.DELIMITER);
335 multiMutate(meta, tableRow, putOfMerged, deleteA, deleteB);
336 } finally {
337 meta.close();
338 }
339 }
340
341
342
343
344
345
346
347
348
349
350
351
352 public static void splitRegion(final CatalogTracker catalogTracker,
353 HRegionInfo parent, HRegionInfo splitA, HRegionInfo splitB,
354 ServerName sn) throws IOException {
355 HTable meta = MetaReader.getMetaHTable(catalogTracker);
356 try {
357 HRegionInfo copyOfParent = new HRegionInfo(parent);
358 copyOfParent.setOffline(true);
359 copyOfParent.setSplit(true);
360
361
362 Put putParent = makePutFromRegionInfo(copyOfParent);
363 addDaughtersToPut(putParent, splitA, splitB);
364
365
366 Put putA = makePutFromRegionInfo(splitA);
367 Put putB = makePutFromRegionInfo(splitB);
368
369 addLocation(putA, sn, 1);
370 addLocation(putB, sn, 1);
371
372 byte[] tableRow = Bytes.toBytes(parent.getRegionNameAsString() + HConstants.DELIMITER);
373 multiMutate(meta, tableRow, putParent, putA, putB);
374 } finally {
375 meta.close();
376 }
377 }
378
379
380
381
382 private static void multiMutate(HTable table, byte[] row, Mutation... mutations) throws IOException {
383 CoprocessorRpcChannel channel = table.coprocessorService(row);
384 MutateRowsRequest.Builder mmrBuilder = MutateRowsRequest.newBuilder();
385 for (Mutation mutation : mutations) {
386 if (mutation instanceof Put) {
387 mmrBuilder.addMutationRequest(ProtobufUtil.toMutation(MutationType.PUT, mutation));
388 } else if (mutation instanceof Delete) {
389 mmrBuilder.addMutationRequest(ProtobufUtil.toMutation(MutationType.DELETE, mutation));
390 } else {
391 throw new DoNotRetryIOException("multi in MetaEditor doesn't support "
392 + mutation.getClass().getName());
393 }
394 }
395
396 MultiRowMutationService.BlockingInterface service =
397 MultiRowMutationService.newBlockingStub(channel);
398 try {
399 service.mutateRows(null, mmrBuilder.build());
400 } catch (ServiceException ex) {
401 ProtobufUtil.toIOException(ex);
402 }
403 }
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422 public static void updateMetaLocation(CatalogTracker catalogTracker,
423 HRegionInfo regionInfo, ServerName sn, long openSeqNum)
424 throws IOException, ConnectException {
425 updateLocation(catalogTracker, regionInfo, sn, openSeqNum);
426 }
427
428
429
430
431
432
433
434
435
436
437
438
439
440 public static void updateRegionLocation(CatalogTracker catalogTracker,
441 HRegionInfo regionInfo, ServerName sn, long updateSeqNum)
442 throws IOException {
443 updateLocation(catalogTracker, regionInfo, sn, updateSeqNum);
444 }
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459 private static void updateLocation(final CatalogTracker catalogTracker,
460 HRegionInfo regionInfo, ServerName sn, long openSeqNum)
461 throws IOException {
462 Put put = new Put(regionInfo.getRegionName());
463 addLocation(put, sn, openSeqNum);
464 putToCatalogTable(catalogTracker, put);
465 LOG.info("Updated row " + regionInfo.getRegionNameAsString() +
466 " with server=" + sn);
467 }
468
469
470
471
472
473
474
475 public static void deleteRegion(CatalogTracker catalogTracker,
476 HRegionInfo regionInfo)
477 throws IOException {
478 Delete delete = new Delete(regionInfo.getRegionName());
479 deleteFromMetaTable(catalogTracker, delete);
480 LOG.info("Deleted " + regionInfo.getRegionNameAsString());
481 }
482
483
484
485
486
487
488
489 public static void deleteRegions(CatalogTracker catalogTracker,
490 List<HRegionInfo> regionsInfo) throws IOException {
491 List<Delete> deletes = new ArrayList<Delete>(regionsInfo.size());
492 for (HRegionInfo hri: regionsInfo) {
493 deletes.add(new Delete(hri.getRegionName()));
494 }
495 deleteFromMetaTable(catalogTracker, deletes);
496 LOG.info("Deleted " + regionsInfo);
497 }
498
499
500
501
502
503
504
505
506 public static void mutateRegions(CatalogTracker catalogTracker,
507 final List<HRegionInfo> regionsToRemove, final List<HRegionInfo> regionsToAdd)
508 throws IOException {
509 List<Mutation> mutation = new ArrayList<Mutation>();
510 if (regionsToRemove != null) {
511 for (HRegionInfo hri: regionsToRemove) {
512 mutation.add(new Delete(hri.getRegionName()));
513 }
514 }
515 if (regionsToAdd != null) {
516 for (HRegionInfo hri: regionsToAdd) {
517 mutation.add(makePutFromRegionInfo(hri));
518 }
519 }
520 mutateMetaTable(catalogTracker, mutation);
521 if (regionsToRemove != null && regionsToRemove.size() > 0) {
522 LOG.debug("Deleted " + regionsToRemove);
523 }
524 if (regionsToAdd != null && regionsToAdd.size() > 0) {
525 LOG.debug("Added " + regionsToAdd);
526 }
527 }
528
529
530
531
532
533
534
535 public static void overwriteRegions(CatalogTracker catalogTracker,
536 List<HRegionInfo> regionInfos) throws IOException {
537 deleteRegions(catalogTracker, regionInfos);
538
539
540
541
542 Threads.sleep(20);
543 addRegionsToMeta(catalogTracker, regionInfos);
544 LOG.info("Overwritten " + regionInfos);
545 }
546
547
548
549
550
551
552
553 public static void deleteMergeQualifiers(CatalogTracker catalogTracker,
554 final HRegionInfo mergedRegion) throws IOException {
555 Delete delete = new Delete(mergedRegion.getRegionName());
556 delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER);
557 delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER);
558 deleteFromMetaTable(catalogTracker, delete);
559 LOG.info("Deleted references in merged region "
560 + mergedRegion.getRegionNameAsString() + ", qualifier="
561 + Bytes.toStringBinary(HConstants.MERGEA_QUALIFIER) + " and qualifier="
562 + Bytes.toStringBinary(HConstants.MERGEB_QUALIFIER));
563 }
564
565 private static Put addRegionInfo(final Put p, final HRegionInfo hri)
566 throws IOException {
567 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
568 hri.toByteArray());
569 return p;
570 }
571
572 private static Put addLocation(final Put p, final ServerName sn, long openSeqNum) {
573
574
575 long now = EnvironmentEdgeManager.currentTimeMillis();
576 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, now,
577 Bytes.toBytes(sn.getHostAndPort()));
578 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, now,
579 Bytes.toBytes(sn.getStartcode()));
580 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.SEQNUM_QUALIFIER, now,
581 Bytes.toBytes(openSeqNum));
582 return p;
583 }
584 }