View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   * http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.Collection;
25  import java.util.HashMap;
26  import java.util.List;
27  import java.util.Map;
28  import java.util.NavigableSet;
29  import java.util.UUID;
30  import java.util.concurrent.ArrayBlockingQueue;
31  import java.util.concurrent.BlockingQueue;
32  import java.util.concurrent.ConcurrentHashMap;
33  import java.util.concurrent.ConcurrentMap;
34  import java.util.regex.Matcher;
35  
36  import org.apache.commons.collections.map.AbstractReferenceMap;
37  import org.apache.commons.collections.map.ReferenceMap;
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
41  import org.apache.hadoop.hbase.classification.InterfaceAudience;
42  import org.apache.hadoop.hbase.classification.InterfaceStability;
43  import org.apache.hadoop.conf.Configuration;
44  import org.apache.hadoop.fs.FileSystem;
45  import org.apache.hadoop.fs.Path;
46  import org.apache.hadoop.hbase.Cell;
47  import org.apache.hadoop.hbase.Coprocessor;
48  import org.apache.hadoop.hbase.CoprocessorEnvironment;
49  import org.apache.hadoop.hbase.HBaseConfiguration;
50  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
51  import org.apache.hadoop.hbase.HConstants;
52  import org.apache.hadoop.hbase.HRegionInfo;
53  import org.apache.hadoop.hbase.HTableDescriptor;
54  import org.apache.hadoop.hbase.client.Append;
55  import org.apache.hadoop.hbase.client.Delete;
56  import org.apache.hadoop.hbase.client.Durability;
57  import org.apache.hadoop.hbase.client.Get;
58  import org.apache.hadoop.hbase.client.Increment;
59  import org.apache.hadoop.hbase.client.Mutation;
60  import org.apache.hadoop.hbase.client.Put;
61  import org.apache.hadoop.hbase.client.Result;
62  import org.apache.hadoop.hbase.client.Scan;
63  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
64  import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
65  import org.apache.hadoop.hbase.coprocessor.EndpointObserver;
66  import org.apache.hadoop.hbase.coprocessor.ObserverContext;
67  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
68  import org.apache.hadoop.hbase.coprocessor.RegionObserver;
69  import org.apache.hadoop.hbase.coprocessor.RegionObserver.MutationType;
70  import org.apache.hadoop.hbase.filter.ByteArrayComparable;
71  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
72  import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
73  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
74  import org.apache.hadoop.hbase.io.Reference;
75  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
76  import org.apache.hadoop.hbase.regionserver.HRegion.Operation;
77  import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
78  import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
79  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
80  import org.apache.hadoop.hbase.util.Bytes;
81  import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
82  import org.apache.hadoop.hbase.util.Pair;
83  
84  import com.google.common.collect.ImmutableList;
85  import com.google.common.collect.Lists;
86  import com.google.protobuf.Message;
87  import com.google.protobuf.Service;
88  
89  /**
90   * Implements the coprocessor environment and runtime support for coprocessors
91   * loaded within a {@link HRegion}.
92   */
93  @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
94  @InterfaceStability.Evolving
95  public class RegionCoprocessorHost
96      extends CoprocessorHost<RegionCoprocessorHost.RegionEnvironment> {
97  
98    private static final Log LOG = LogFactory.getLog(RegionCoprocessorHost.class);
99    // The shared data map
100   private static ReferenceMap sharedDataMap =
101       new ReferenceMap(AbstractReferenceMap.HARD, AbstractReferenceMap.WEAK);
102 
103   /**
104    * Encapsulation of the environment of each coprocessor
105    */
106   static class RegionEnvironment extends CoprocessorHost.Environment
107       implements RegionCoprocessorEnvironment {
108 
109     private HRegion region;
110     private RegionServerServices rsServices;
111     ConcurrentMap<String, Object> sharedData;
112     private static final int LATENCY_BUFFER_SIZE = 100;
113     private final BlockingQueue<Long> coprocessorTimeNanos = new ArrayBlockingQueue<Long>(
114         LATENCY_BUFFER_SIZE);
115 
116     /**
117      * Constructor
118      * @param impl the coprocessor instance
119      * @param priority chaining priority
120      */
121     public RegionEnvironment(final Coprocessor impl, final int priority,
122         final int seq, final Configuration conf, final HRegion region,
123         final RegionServerServices services, final ConcurrentMap<String, Object> sharedData) {
124       super(impl, priority, seq, conf);
125       this.region = region;
126       this.rsServices = services;
127       this.sharedData = sharedData;
128     }
129 
130     /** @return the region */
131     @Override
132     public HRegion getRegion() {
133       return region;
134     }
135 
136     /** @return reference to the region server services */
137     @Override
138     public RegionServerServices getRegionServerServices() {
139       return rsServices;
140     }
141 
142     public void shutdown() {
143       super.shutdown();
144     }
145 
146     @Override
147     public ConcurrentMap<String, Object> getSharedData() {
148       return sharedData;
149     }
150 
151     public void offerExecutionLatency(long latencyNanos) {
152       coprocessorTimeNanos.offer(latencyNanos);
153     }
154 
155     public Collection<Long> getExecutionLatenciesNanos() {
156       final List<Long> latencies = Lists.newArrayListWithCapacity(coprocessorTimeNanos.size());
157       coprocessorTimeNanos.drainTo(latencies);
158       return latencies;
159     }
160 
161     @Override
162     public HRegionInfo getRegionInfo() {
163       return region.getRegionInfo();
164     }
165 
166   }
167 
168   static class TableCoprocessorAttribute {
169     private Path path;
170     private String className;
171     private int priority;
172     private Configuration conf;
173 
174     public TableCoprocessorAttribute(Path path, String className, int priority,
175         Configuration conf) {
176       this.path = path;
177       this.className = className;
178       this.priority = priority;
179       this.conf = conf;
180     }
181 
182     public Path getPath() {
183       return path;
184     }
185 
186     public String getClassName() {
187       return className;
188     }
189 
190     public int getPriority() {
191       return priority;
192     }
193 
194     public Configuration getConf() {
195       return conf;
196     }
197   }
198 
199   /** The region server services */
200   RegionServerServices rsServices;
201   /** The region */
202   HRegion region;
203 
204   /**
205    * Constructor
206    * @param region the region
207    * @param rsServices interface to available region server functionality
208    * @param conf the configuration
209    */
210   public RegionCoprocessorHost(final HRegion region,
211       final RegionServerServices rsServices, final Configuration conf) {
212     super(rsServices);
213     this.conf = conf;
214     this.rsServices = rsServices;
215     this.region = region;
216     this.pathPrefix = Integer.toString(this.region.getRegionInfo().hashCode());
217 
218     // load system default cp's from configuration.
219     loadSystemCoprocessors(conf, REGION_COPROCESSOR_CONF_KEY);
220 
221     // load system default cp's for user tables from configuration.
222     if (!region.getRegionInfo().getTable().isSystemTable()) {
223       loadSystemCoprocessors(conf, USER_REGION_COPROCESSOR_CONF_KEY);
224     }
225 
226     // load Coprocessor From HDFS
227     loadTableCoprocessors(conf);
228   }
229 
230   static List<TableCoprocessorAttribute> getTableCoprocessorAttrsFromSchema(Configuration conf,
231       HTableDescriptor htd) {
232     List<TableCoprocessorAttribute> result = Lists.newArrayList();
233     for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e: htd.getValues().entrySet()) {
234       String key = Bytes.toString(e.getKey().get()).trim();
235       if (HConstants.CP_HTD_ATTR_KEY_PATTERN.matcher(key).matches()) {
236         String spec = Bytes.toString(e.getValue().get()).trim();
237         // found one
238         try {
239           Matcher matcher = HConstants.CP_HTD_ATTR_VALUE_PATTERN.matcher(spec);
240           if (matcher.matches()) {
241             // jar file path can be empty if the cp class can be loaded
242             // from class loader.
243             Path path = matcher.group(1).trim().isEmpty() ?
244                 null : new Path(matcher.group(1).trim());
245             String className = matcher.group(2).trim();
246             if (className.isEmpty()) {
247               LOG.error("Malformed table coprocessor specification: key=" +
248                 key + ", spec: " + spec);
249               continue;
250             }
251             int priority = matcher.group(3).trim().isEmpty() ?
252                 Coprocessor.PRIORITY_USER : Integer.valueOf(matcher.group(3));
253             String cfgSpec = null;
254             try {
255               cfgSpec = matcher.group(4);
256             } catch (IndexOutOfBoundsException ex) {
257               // ignore
258             }
259             Configuration ourConf;
260             if (cfgSpec != null) {
261               cfgSpec = cfgSpec.substring(cfgSpec.indexOf('|') + 1);
262               // do an explicit deep copy of the passed configuration
263               ourConf = new Configuration(false);
264               HBaseConfiguration.merge(ourConf, conf);
265               Matcher m = HConstants.CP_HTD_ATTR_VALUE_PARAM_PATTERN.matcher(cfgSpec);
266               while (m.find()) {
267                 ourConf.set(m.group(1), m.group(2));
268               }
269             } else {
270               ourConf = conf;
271             }
272             result.add(new TableCoprocessorAttribute(path, className, priority, ourConf));
273           } else {
274             LOG.error("Malformed table coprocessor specification: key=" + key +
275               ", spec: " + spec);
276           }
277         } catch (Exception ioe) {
278           LOG.error("Malformed table coprocessor specification: key=" + key +
279             ", spec: " + spec);
280         }
281       }
282     }
283     return result;
284   }
285 
286   /**
287    * Sanity check the table coprocessor attributes of the supplied schema. Will
288    * throw an exception if there is a problem.
289    * @param conf
290    * @param htd
291    * @throws IOException
292    */
293   public static void testTableCoprocessorAttrs(final Configuration conf,
294       final HTableDescriptor htd) throws IOException {
295     String pathPrefix = UUID.randomUUID().toString();
296     for (TableCoprocessorAttribute attr: getTableCoprocessorAttrsFromSchema(conf, htd)) {
297       if (attr.getPriority() < 0) {
298         throw new IOException("Priority for coprocessor " + attr.getClassName() +
299           " cannot be less than 0");
300       }
301       ClassLoader old = Thread.currentThread().getContextClassLoader();
302       try {
303         ClassLoader cl;
304         if (attr.getPath() != null) {
305           cl = CoprocessorClassLoader.getClassLoader(attr.getPath(),
306             CoprocessorHost.class.getClassLoader(), pathPrefix, conf);
307         } else {
308           cl = CoprocessorHost.class.getClassLoader();
309         }
310         Thread.currentThread().setContextClassLoader(cl);
311         cl.loadClass(attr.getClassName());
312       } catch (ClassNotFoundException e) {
313         throw new IOException("Class " + attr.getClassName() + " cannot be loaded", e);
314       } finally {
315         Thread.currentThread().setContextClassLoader(old);
316       }
317     }
318   }
319 
320   void loadTableCoprocessors(final Configuration conf) {
321     boolean coprocessorsEnabled = conf.getBoolean(COPROCESSORS_ENABLED_CONF_KEY,
322       DEFAULT_COPROCESSORS_ENABLED);
323     boolean tableCoprocessorsEnabled = conf.getBoolean(USER_COPROCESSORS_ENABLED_CONF_KEY,
324       DEFAULT_USER_COPROCESSORS_ENABLED);
325     if (!(coprocessorsEnabled && tableCoprocessorsEnabled)) {
326       return;
327     }
328 
329     // scan the table attributes for coprocessor load specifications
330     // initialize the coprocessors
331     List<RegionEnvironment> configured = new ArrayList<RegionEnvironment>();
332     for (TableCoprocessorAttribute attr: getTableCoprocessorAttrsFromSchema(conf, 
333         region.getTableDesc())) {
334       // Load encompasses classloading and coprocessor initialization
335       try {
336         RegionEnvironment env = load(attr.getPath(), attr.getClassName(), attr.getPriority(),
337           attr.getConf());
338         configured.add(env);
339         LOG.info("Loaded coprocessor " + attr.getClassName() + " from HTD of " +
340             region.getTableDesc().getTableName().getNameAsString() + " successfully.");
341       } catch (Throwable t) {
342         // Coprocessor failed to load, do we abort on error?
343         if (conf.getBoolean(ABORT_ON_ERROR_KEY, DEFAULT_ABORT_ON_ERROR)) {
344           abortServer(attr.getClassName(), t);
345         } else {
346           LOG.error("Failed to load coprocessor " + attr.getClassName(), t);
347         }
348       }
349     }
350     // add together to coprocessor set for COW efficiency
351     coprocessors.addAll(configured);
352   }
353 
354   @Override
355   public RegionEnvironment createEnvironment(Class<?> implClass,
356       Coprocessor instance, int priority, int seq, Configuration conf) {
357     // Check if it's an Endpoint.
358     // Due to current dynamic protocol design, Endpoint
359     // uses a different way to be registered and executed.
360     // It uses a visitor pattern to invoke registered Endpoint
361     // method.
362     for (Class<?> c : implClass.getInterfaces()) {
363       if (CoprocessorService.class.isAssignableFrom(c)) {
364         region.registerService( ((CoprocessorService)instance).getService() );
365       }
366     }
367     ConcurrentMap<String, Object> classData;
368     // make sure only one thread can add maps
369     synchronized (sharedDataMap) {
370       // as long as at least one RegionEnvironment holds on to its classData it will
371       // remain in this map
372       classData = (ConcurrentMap<String, Object>)sharedDataMap.get(implClass.getName());
373       if (classData == null) {
374         classData = new ConcurrentHashMap<String, Object>();
375         sharedDataMap.put(implClass.getName(), classData);
376       }
377     }
378     return new RegionEnvironment(instance, priority, seq, conf, region,
379         rsServices, classData);
380   }
381 
382   /**
383    * HBASE-4014 : This is used by coprocessor hooks which are not declared to throw exceptions.
384    *
385    * For example, {@link
386    * org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost#preOpen()} and
387    * {@link org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost#postOpen()} are such hooks.
388    *
389    * See also
390    * {@link org.apache.hadoop.hbase.master.MasterCoprocessorHost#handleCoprocessorThrowable(
391    *    CoprocessorEnvironment, Throwable)}
392    * @param env The coprocessor that threw the exception.
393    * @param e The exception that was thrown.
394    */
395   private void handleCoprocessorThrowableNoRethrow(
396       final CoprocessorEnvironment env, final Throwable e) {
397     try {
398       handleCoprocessorThrowable(env,e);
399     } catch (IOException ioe) {
400       // We cannot throw exceptions from the caller hook, so ignore.
401       LOG.warn(
402         "handleCoprocessorThrowable() threw an IOException while attempting to handle Throwable " +
403         e + ". Ignoring.",e);
404     }
405   }
406 
407   /**
408    * Invoked before a region open.
409    *
410    * @throws IOException Signals that an I/O exception has occurred.
411    */
412   public void preOpen() throws IOException {
413     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
414       @Override
415       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
416           throws IOException {
417         oserver.preOpen(ctx);
418       }
419     });
420   }
421 
422   /**
423    * Invoked after a region open
424    */
425   public void postOpen() {
426     try {
427       execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
428         @Override
429         public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
430             throws IOException {
431           oserver.postOpen(ctx);
432         }
433       });
434     } catch (IOException e) {
435       LOG.warn(e);
436     }
437   }
438 
439   /**
440    * Invoked after log replay on region
441    */
442   public void postLogReplay() {
443     try {
444       execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
445         @Override
446         public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
447             throws IOException {
448           oserver.postLogReplay(ctx);
449         }
450       });
451     } catch (IOException e) {
452       LOG.warn(e);
453     }
454   }
455 
456   /**
457    * Invoked before a region is closed
458    * @param abortRequested true if the server is aborting
459    */
460   public void preClose(final boolean abortRequested) throws IOException {
461     execOperation(false, new RegionOperation() {
462       @Override
463       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
464           throws IOException {
465         oserver.preClose(ctx, abortRequested);
466       }
467     });
468   }
469 
470   /**
471    * Invoked after a region is closed
472    * @param abortRequested true if the server is aborting
473    */
474   public void postClose(final boolean abortRequested) {
475     try {
476       execOperation(false, new RegionOperation() {
477         @Override
478         public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
479             throws IOException {
480           oserver.postClose(ctx, abortRequested);
481         }
482         public void postEnvCall(RegionEnvironment env) {
483           shutdown(env);
484         }
485       });
486     } catch (IOException e) {
487       LOG.warn(e);
488     }
489   }
490 
491   /**
492    * See
493    * {@link RegionObserver#preCompactScannerOpen(ObserverContext, Store, List, ScanType, long, InternalScanner, CompactionRequest)}
494    */
495   public InternalScanner preCompactScannerOpen(final Store store,
496       final List<StoreFileScanner> scanners, final ScanType scanType, final long earliestPutTs,
497       final CompactionRequest request) throws IOException {
498     return execOperationWithResult(null,
499         coprocessors.isEmpty() ? null : new RegionOperationWithResult<InternalScanner>() {
500       @Override
501       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
502           throws IOException {
503         setResult(oserver.preCompactScannerOpen(ctx, store, scanners, scanType,
504           earliestPutTs, getResult(), request));
505       }
506     });
507   }
508 
509   /**
510    * Called prior to selecting the {@link StoreFile}s for compaction from the list of currently
511    * available candidates.
512    * @param store The store where compaction is being requested
513    * @param candidates The currently available store files
514    * @param request custom compaction request
515    * @return If {@code true}, skip the normal selection process and use the current list
516    * @throws IOException
517    */
518   public boolean preCompactSelection(final Store store, final List<StoreFile> candidates,
519       final CompactionRequest request) throws IOException {
520     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
521       @Override
522       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
523           throws IOException {
524         oserver.preCompactSelection(ctx, store, candidates, request);
525       }
526     });
527   }
528 
529   /**
530    * Called after the {@link StoreFile}s to be compacted have been selected from the available
531    * candidates.
532    * @param store The store where compaction is being requested
533    * @param selected The store files selected to compact
534    * @param request custom compaction
535    */
536   public void postCompactSelection(final Store store, final ImmutableList<StoreFile> selected,
537       final CompactionRequest request) {
538     try {
539       execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
540         @Override
541         public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
542             throws IOException {
543           oserver.postCompactSelection(ctx, store, selected, request);
544         }
545       });
546     } catch (IOException e) {
547       LOG.warn(e);
548     }
549   }
550 
551   /**
552    * Called prior to rewriting the store files selected for compaction
553    * @param store the store being compacted
554    * @param scanner the scanner used to read store data during compaction
555    * @param scanType type of Scan
556    * @param request the compaction that will be executed
557    * @throws IOException
558    */
559   public InternalScanner preCompact(final Store store, final InternalScanner scanner,
560       final ScanType scanType, final CompactionRequest request) throws IOException {
561     return execOperationWithResult(false, scanner,
562         coprocessors.isEmpty() ? null : new RegionOperationWithResult<InternalScanner>() {
563       @Override
564       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
565           throws IOException {
566         setResult(oserver.preCompact(ctx, store, getResult(), scanType, request));
567       }
568     });
569   }
570 
571   /**
572    * Called after the store compaction has completed.
573    * @param store the store being compacted
574    * @param resultFile the new store file written during compaction
575    * @param request the compaction that is being executed
576    * @throws IOException
577    */
578   public void postCompact(final Store store, final StoreFile resultFile,
579       final CompactionRequest request) throws IOException {
580     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
581       @Override
582       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
583           throws IOException {
584         oserver.postCompact(ctx, store, resultFile, request);
585       }
586     });
587   }
588 
589   /**
590    * Invoked before a memstore flush
591    * @throws IOException
592    */
593   public InternalScanner preFlush(final Store store, final InternalScanner scanner)
594       throws IOException {
595     return execOperationWithResult(false, scanner,
596         coprocessors.isEmpty() ? null : new RegionOperationWithResult<InternalScanner>() {
597       @Override
598       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
599           throws IOException {
600         setResult(oserver.preFlush(ctx, store, getResult()));
601       }
602     });
603   }
604 
605   /**
606    * Invoked before a memstore flush
607    * @throws IOException
608    */
609   public void preFlush() throws IOException {
610     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
611       @Override
612       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
613           throws IOException {
614         oserver.preFlush(ctx);
615       }
616     });
617   }
618 
619   /**
620    * See
621    * {@link RegionObserver#preFlushScannerOpen(ObserverContext,
622    *    Store, KeyValueScanner, InternalScanner)}
623    */
624   public InternalScanner preFlushScannerOpen(final Store store,
625       final KeyValueScanner memstoreScanner) throws IOException {
626     return execOperationWithResult(null,
627         coprocessors.isEmpty() ? null : new RegionOperationWithResult<InternalScanner>() {
628       @Override
629       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
630           throws IOException {
631         setResult(oserver.preFlushScannerOpen(ctx, store, memstoreScanner, getResult()));
632       }
633     });
634   }
635 
636   /**
637    * Invoked after a memstore flush
638    * @throws IOException
639    */
640   public void postFlush() throws IOException {
641     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
642       @Override
643       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
644           throws IOException {
645         oserver.postFlush(ctx);
646       }
647     });
648   }
649 
650   /**
651    * Invoked after a memstore flush
652    * @throws IOException
653    */
654   public void postFlush(final Store store, final StoreFile storeFile) throws IOException {
655     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
656       @Override
657       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
658           throws IOException {
659         oserver.postFlush(ctx, store, storeFile);
660       }
661     });
662   }
663 
664   /**
665    * Invoked just before a split
666    * @throws IOException
667    */
668   // TODO: Deprecate this
669   public void preSplit() throws IOException {
670     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
671       @Override
672       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
673           throws IOException {
674         oserver.preSplit(ctx);
675       }
676     });
677   }
678 
679   /**
680    * Invoked just before a split
681    * @throws IOException
682    */
683   public void preSplit(final byte[] splitRow) throws IOException {
684     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
685       @Override
686       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
687           throws IOException {
688         oserver.preSplit(ctx, splitRow);
689       }
690     });
691   }
692 
693   /**
694    * Invoked just after a split
695    * @param l the new left-hand daughter region
696    * @param r the new right-hand daughter region
697    * @throws IOException
698    */
699   public void postSplit(final HRegion l, final HRegion r) throws IOException {
700     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
701       @Override
702       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
703           throws IOException {
704         oserver.postSplit(ctx, l, r);
705       }
706     });
707   }
708 
709   public boolean preSplitBeforePONR(final byte[] splitKey,
710       final List<Mutation> metaEntries) throws IOException {
711     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
712       @Override
713       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
714           throws IOException {
715         oserver.preSplitBeforePONR(ctx, splitKey, metaEntries);
716       }
717     });
718   }
719 
720   public void preSplitAfterPONR() throws IOException {
721     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
722       @Override
723       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
724           throws IOException {
725         oserver.preSplitAfterPONR(ctx);
726       }
727     });
728   }
729 
730   /**
731    * Invoked just before the rollback of a failed split is started
732    * @throws IOException
733    */
734   public void preRollBackSplit() throws IOException {
735     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
736       @Override
737       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
738           throws IOException {
739         oserver.preRollBackSplit(ctx);
740       }
741     });
742   }
743 
744   /**
745    * Invoked just after the rollback of a failed split is done
746    * @throws IOException
747    */
748   public void postRollBackSplit() throws IOException {
749     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
750       @Override
751       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
752           throws IOException {
753         oserver.postRollBackSplit(ctx);
754       }
755     });
756   }
757 
758   /**
759    * Invoked after a split is completed irrespective of a failure or success.
760    * @throws IOException
761    */
762   public void postCompleteSplit() throws IOException {
763     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
764       @Override
765       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
766           throws IOException {
767         oserver.postCompleteSplit(ctx);
768       }
769     });
770   }
771 
772   // RegionObserver support
773 
774   /**
775    * @param row the row key
776    * @param family the family
777    * @param result the result set from the region
778    * @return true if default processing should be bypassed
779    * @exception IOException Exception
780    */
781   public boolean preGetClosestRowBefore(final byte[] row, final byte[] family,
782       final Result result) throws IOException {
783     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
784       @Override
785       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
786           throws IOException {
787         oserver.preGetClosestRowBefore(ctx, row, family, result);
788       }
789     });
790   }
791 
792   /**
793    * @param row the row key
794    * @param family the family
795    * @param result the result set from the region
796    * @exception IOException Exception
797    */
798   public void postGetClosestRowBefore(final byte[] row, final byte[] family,
799       final Result result) throws IOException {
800     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
801       @Override
802       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
803           throws IOException {
804         oserver.postGetClosestRowBefore(ctx, row, family, result);
805       }
806     });
807   }
808 
809   /**
810    * @param get the Get request
811    * @return true if default processing should be bypassed
812    * @exception IOException Exception
813    */
814   public boolean preGet(final Get get, final List<Cell> results)
815       throws IOException {
816     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
817       @Override
818       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
819           throws IOException {
820         oserver.preGetOp(ctx, get, results);
821       }
822     });
823   }
824 
825   /**
826    * @param get the Get request
827    * @param results the result sett
828    * @exception IOException Exception
829    */
830   public void postGet(final Get get, final List<Cell> results)
831       throws IOException {
832     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
833       @Override
834       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
835           throws IOException {
836         oserver.postGetOp(ctx, get, results);
837       }
838     });
839   }
840 
841   /**
842    * @param get the Get request
843    * @return true or false to return to client if bypassing normal operation,
844    * or null otherwise
845    * @exception IOException Exception
846    */
847   public Boolean preExists(final Get get) throws IOException {
848     return execOperationWithResult(true, false,
849         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
850       @Override
851       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
852           throws IOException {
853         setResult(oserver.preExists(ctx, get, getResult()));
854       }
855     });
856   }
857 
858   /**
859    * @param get the Get request
860    * @param exists the result returned by the region server
861    * @return the result to return to the client
862    * @exception IOException Exception
863    */
864   public boolean postExists(final Get get, boolean exists)
865       throws IOException {
866     return execOperationWithResult(exists,
867         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
868       @Override
869       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
870           throws IOException {
871         setResult(oserver.postExists(ctx, get, getResult()));
872       }
873     });
874   }
875 
876   /**
877    * @param put The Put object
878    * @param edit The WALEdit object.
879    * @param durability The durability used
880    * @return true if default processing should be bypassed
881    * @exception IOException Exception
882    */
883   public boolean prePut(final Put put, final WALEdit edit, final Durability durability)
884       throws IOException {
885     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
886       @Override
887       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
888           throws IOException {
889         oserver.prePut(ctx, put, edit, durability);
890       }
891     });
892   }
893 
894   /**
895    * @param mutation - the current mutation
896    * @param kv - the current cell
897    * @param byteNow - current timestamp in bytes
898    * @param get - the get that could be used
899    * Note that the get only does not specify the family and qualifier that should be used
900    * @return true if default processing should be bypassed
901    * @exception IOException
902    *              Exception
903    */
904   public boolean prePrepareTimeStampForDeleteVersion(final Mutation mutation,
905       final Cell kv, final byte[] byteNow, final Get get) throws IOException {
906     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
907       @Override
908       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
909           throws IOException {
910         oserver.prePrepareTimeStampForDeleteVersion(ctx, mutation, kv, byteNow, get);
911       }
912     });
913   }
914 
915   /**
916    * @param put The Put object
917    * @param edit The WALEdit object.
918    * @param durability The durability used
919    * @exception IOException Exception
920    */
921   public void postPut(final Put put, final WALEdit edit, final Durability durability)
922       throws IOException {
923     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
924       @Override
925       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
926           throws IOException {
927         oserver.postPut(ctx, put, edit, durability);
928       }
929     });
930   }
931 
932   /**
933    * @param delete The Delete object
934    * @param edit The WALEdit object.
935    * @param durability The durability used
936    * @return true if default processing should be bypassed
937    * @exception IOException Exception
938    */
939   public boolean preDelete(final Delete delete, final WALEdit edit, final Durability durability)
940       throws IOException {
941     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
942       @Override
943       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
944           throws IOException {
945         oserver.preDelete(ctx, delete, edit, durability);
946       }
947     });
948   }
949 
950   /**
951    * @param delete The Delete object
952    * @param edit The WALEdit object.
953    * @param durability The durability used
954    * @exception IOException Exception
955    */
956   public void postDelete(final Delete delete, final WALEdit edit, final Durability durability)
957       throws IOException {
958     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
959       @Override
960       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
961           throws IOException {
962         oserver.postDelete(ctx, delete, edit, durability);
963       }
964     });
965   }
966 
967   /**
968    * @param miniBatchOp
969    * @return true if default processing should be bypassed
970    * @throws IOException
971    */
972   public boolean preBatchMutate(
973       final MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {
974     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
975       @Override
976       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
977           throws IOException {
978         oserver.preBatchMutate(ctx, miniBatchOp);
979       }
980     });
981   }
982 
983   /**
984    * @param miniBatchOp
985    * @throws IOException
986    */
987   public void postBatchMutate(
988       final MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {
989     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
990       @Override
991       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
992           throws IOException {
993         oserver.postBatchMutate(ctx, miniBatchOp);
994       }
995     });
996   }
997 
998   public void postBatchMutateIndispensably(
999       final MiniBatchOperationInProgress<Mutation> miniBatchOp, final boolean success)
1000       throws IOException {
1001     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
1002       @Override
1003       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1004           throws IOException {
1005         oserver.postBatchMutateIndispensably(ctx, miniBatchOp, success);
1006       }
1007     });
1008   }
1009 
1010   /**
1011    * @param row row to check
1012    * @param family column family
1013    * @param qualifier column qualifier
1014    * @param compareOp the comparison operation
1015    * @param comparator the comparator
1016    * @param put data to put if check succeeds
1017    * @return true or false to return to client if default processing should
1018    * be bypassed, or null otherwise
1019    * @throws IOException e
1020    */
1021   public Boolean preCheckAndPut(final byte [] row, final byte [] family,
1022       final byte [] qualifier, final CompareOp compareOp,
1023       final ByteArrayComparable comparator, final Put put)
1024       throws IOException {
1025     return execOperationWithResult(true, false,
1026         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
1027       @Override
1028       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1029           throws IOException {
1030         setResult(oserver.preCheckAndPut(ctx, row, family, qualifier,
1031           compareOp, comparator, put, getResult()));
1032       }
1033     });
1034   }
1035 
1036   /**
1037    * @param row row to check
1038    * @param family column family
1039    * @param qualifier column qualifier
1040    * @param compareOp the comparison operation
1041    * @param comparator the comparator
1042    * @param put data to put if check succeeds
1043    * @return true or false to return to client if default processing should
1044    * be bypassed, or null otherwise
1045    * @throws IOException e
1046    */
1047   public Boolean preCheckAndPutAfterRowLock(final byte[] row, final byte[] family,
1048       final byte[] qualifier, final CompareOp compareOp, final ByteArrayComparable comparator,
1049       final Put put) throws IOException {
1050     return execOperationWithResult(true, false,
1051         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
1052       @Override
1053       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1054           throws IOException {
1055         setResult(oserver.preCheckAndPutAfterRowLock(ctx, row, family, qualifier,
1056           compareOp, comparator, put, getResult()));
1057       }
1058     });
1059   }
1060 
1061   /**
1062    * @param row row to check
1063    * @param family column family
1064    * @param qualifier column qualifier
1065    * @param compareOp the comparison operation
1066    * @param comparator the comparator
1067    * @param put data to put if check succeeds
1068    * @throws IOException e
1069    */
1070   public boolean postCheckAndPut(final byte [] row, final byte [] family,
1071       final byte [] qualifier, final CompareOp compareOp,
1072       final ByteArrayComparable comparator, final Put put,
1073       boolean result) throws IOException {
1074     return execOperationWithResult(result,
1075         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
1076       @Override
1077       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1078           throws IOException {
1079         setResult(oserver.postCheckAndPut(ctx, row, family, qualifier,
1080           compareOp, comparator, put, getResult()));
1081       }
1082     });
1083   }
1084 
1085   /**
1086    * @param row row to check
1087    * @param family column family
1088    * @param qualifier column qualifier
1089    * @param compareOp the comparison operation
1090    * @param comparator the comparator
1091    * @param delete delete to commit if check succeeds
1092    * @return true or false to return to client if default processing should
1093    * be bypassed, or null otherwise
1094    * @throws IOException e
1095    */
1096   public Boolean preCheckAndDelete(final byte [] row, final byte [] family,
1097       final byte [] qualifier, final CompareOp compareOp,
1098       final ByteArrayComparable comparator, final Delete delete)
1099       throws IOException {
1100     return execOperationWithResult(true, false,
1101         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
1102       @Override
1103       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1104           throws IOException {
1105         setResult(oserver.preCheckAndDelete(ctx, row, family,
1106             qualifier, compareOp, comparator, delete, getResult()));
1107       }
1108     });
1109   }
1110 
1111   /**
1112    * @param row row to check
1113    * @param family column family
1114    * @param qualifier column qualifier
1115    * @param compareOp the comparison operation
1116    * @param comparator the comparator
1117    * @param delete delete to commit if check succeeds
1118    * @return true or false to return to client if default processing should
1119    * be bypassed, or null otherwise
1120    * @throws IOException e
1121    */
1122   public Boolean preCheckAndDeleteAfterRowLock(final byte[] row, final byte[] family,
1123       final byte[] qualifier, final CompareOp compareOp, final ByteArrayComparable comparator,
1124       final Delete delete) throws IOException {
1125     return execOperationWithResult(true, false,
1126         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
1127       @Override
1128       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1129           throws IOException {
1130         setResult(oserver.preCheckAndDeleteAfterRowLock(ctx, row,
1131               family, qualifier, compareOp, comparator, delete, getResult()));
1132       }
1133     });
1134   }
1135 
1136   /**
1137    * @param row row to check
1138    * @param family column family
1139    * @param qualifier column qualifier
1140    * @param compareOp the comparison operation
1141    * @param comparator the comparator
1142    * @param delete delete to commit if check succeeds
1143    * @throws IOException e
1144    */
1145   public boolean postCheckAndDelete(final byte [] row, final byte [] family,
1146       final byte [] qualifier, final CompareOp compareOp,
1147       final ByteArrayComparable comparator, final Delete delete,
1148       boolean result) throws IOException {
1149     return execOperationWithResult(result,
1150         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
1151       @Override
1152       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1153           throws IOException {
1154         setResult(oserver.postCheckAndDelete(ctx, row, family,
1155             qualifier, compareOp, comparator, delete, getResult()));
1156       }
1157     });
1158   }
1159 
1160   /**
1161    * @param append append object
1162    * @return result to return to client if default operation should be
1163    * bypassed, null otherwise
1164    * @throws IOException if an error occurred on the coprocessor
1165    */
1166   public Result preAppend(final Append append) throws IOException {
1167     return execOperationWithResult(true, null,
1168         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Result>() {
1169       @Override
1170       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1171           throws IOException {
1172         setResult(oserver.preAppend(ctx, append));
1173       }
1174     });
1175   }
1176 
1177   /**
1178    * @param append append object
1179    * @return result to return to client if default operation should be
1180    * bypassed, null otherwise
1181    * @throws IOException if an error occurred on the coprocessor
1182    */
1183   public Result preAppendAfterRowLock(final Append append) throws IOException {
1184     return execOperationWithResult(true, null,
1185         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Result>() {
1186       @Override
1187       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1188           throws IOException {
1189         setResult(oserver.preAppendAfterRowLock(ctx, append));
1190       }
1191     });
1192   }
1193 
1194   /**
1195    * @param increment increment object
1196    * @return result to return to client if default operation should be
1197    * bypassed, null otherwise
1198    * @throws IOException if an error occurred on the coprocessor
1199    */
1200   public Result preIncrement(final Increment increment) throws IOException {
1201     return execOperationWithResult(true, null,
1202         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Result>() {
1203       @Override
1204       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1205           throws IOException {
1206         setResult(oserver.preIncrement(ctx, increment));
1207       }
1208     });
1209   }
1210 
1211   /**
1212    * @param increment increment object
1213    * @return result to return to client if default operation should be
1214    * bypassed, null otherwise
1215    * @throws IOException if an error occurred on the coprocessor
1216    */
1217   public Result preIncrementAfterRowLock(final Increment increment) throws IOException {
1218     return execOperationWithResult(true, null,
1219         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Result>() {
1220       @Override
1221       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1222           throws IOException {
1223         setResult(oserver.preIncrementAfterRowLock(ctx, increment));
1224       }
1225     });
1226   }
1227 
1228   /**
1229    * @param append Append object
1230    * @param result the result returned by the append
1231    * @throws IOException if an error occurred on the coprocessor
1232    */
1233   public void postAppend(final Append append, final Result result) throws IOException {
1234     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
1235       @Override
1236       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1237           throws IOException {
1238         oserver.postAppend(ctx, append, result);
1239       }
1240     });
1241   }
1242 
1243   /**
1244    * @param increment increment object
1245    * @param result the result returned by postIncrement
1246    * @throws IOException if an error occurred on the coprocessor
1247    */
1248   public Result postIncrement(final Increment increment, Result result) throws IOException {
1249     return execOperationWithResult(result,
1250         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Result>() {
1251       @Override
1252       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1253           throws IOException {
1254         setResult(oserver.postIncrement(ctx, increment, getResult()));
1255       }
1256     });
1257   }
1258 
1259   /**
1260    * @param scan the Scan specification
1261    * @return scanner id to return to client if default operation should be
1262    * bypassed, false otherwise
1263    * @exception IOException Exception
1264    */
1265   public RegionScanner preScannerOpen(final Scan scan) throws IOException {
1266     return execOperationWithResult(true, null,
1267         coprocessors.isEmpty() ? null : new RegionOperationWithResult<RegionScanner>() {
1268       @Override
1269       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1270           throws IOException {
1271         setResult(oserver.preScannerOpen(ctx, scan, getResult()));
1272       }
1273     });
1274   }
1275 
1276   /**
1277    * See
1278    * {@link RegionObserver#preStoreScannerOpen(ObserverContext,
1279    *    Store, Scan, NavigableSet, KeyValueScanner)}
1280    */
1281   public KeyValueScanner preStoreScannerOpen(final Store store, final Scan scan,
1282       final NavigableSet<byte[]> targetCols) throws IOException {
1283     return execOperationWithResult(null,
1284         coprocessors.isEmpty() ? null : new RegionOperationWithResult<KeyValueScanner>() {
1285       @Override
1286       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1287           throws IOException {
1288         setResult(oserver.preStoreScannerOpen(ctx, store, scan, targetCols, getResult()));
1289       }
1290     });
1291   }
1292 
1293   /**
1294    * @param scan the Scan specification
1295    * @param s the scanner
1296    * @return the scanner instance to use
1297    * @exception IOException Exception
1298    */
1299   public RegionScanner postScannerOpen(final Scan scan, RegionScanner s) throws IOException {
1300     return execOperationWithResult(s,
1301         coprocessors.isEmpty() ? null : new RegionOperationWithResult<RegionScanner>() {
1302       @Override
1303       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1304           throws IOException {
1305         setResult(oserver.postScannerOpen(ctx, scan, getResult()));
1306       }
1307     });
1308   }
1309 
1310   /**
1311    * @param s the scanner
1312    * @param results the result set returned by the region server
1313    * @param limit the maximum number of results to return
1314    * @return 'has next' indication to client if bypassing default behavior, or
1315    * null otherwise
1316    * @exception IOException Exception
1317    */
1318   public Boolean preScannerNext(final InternalScanner s,
1319       final List<Result> results, final int limit) throws IOException {
1320     return execOperationWithResult(true, false,
1321         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
1322       @Override
1323       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1324           throws IOException {
1325         setResult(oserver.preScannerNext(ctx, s, results, limit, getResult()));
1326       }
1327     });
1328   }
1329 
1330   /**
1331    * @param s the scanner
1332    * @param results the result set returned by the region server
1333    * @param limit the maximum number of results to return
1334    * @param hasMore
1335    * @return 'has more' indication to give to client
1336    * @exception IOException Exception
1337    */
1338   public boolean postScannerNext(final InternalScanner s,
1339       final List<Result> results, final int limit, boolean hasMore)
1340       throws IOException {
1341     return execOperationWithResult(hasMore,
1342         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
1343       @Override
1344       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1345           throws IOException {
1346         setResult(oserver.postScannerNext(ctx, s, results, limit, getResult()));
1347       }
1348     });
1349   }
1350 
1351   /**
1352    * This will be called by the scan flow when the current scanned row is being filtered out by the
1353    * filter.
1354    * @param s the scanner
1355    * @param currentRow The current rowkey which got filtered out
1356    * @param offset offset to rowkey
1357    * @param length length of rowkey
1358    * @return whether more rows are available for the scanner or not
1359    * @throws IOException
1360    */
1361   public boolean postScannerFilterRow(final InternalScanner s, final byte[] currentRow,
1362       final int offset, final short length) throws IOException {
1363     return execOperationWithResult(true,
1364         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
1365       @Override
1366       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1367           throws IOException {
1368         setResult(oserver.postScannerFilterRow(ctx, s, currentRow, offset,length, getResult()));
1369       }
1370     });
1371   }
1372 
1373   /**
1374    * @param s the scanner
1375    * @return true if default behavior should be bypassed, false otherwise
1376    * @exception IOException Exception
1377    */
1378   public boolean preScannerClose(final InternalScanner s) throws IOException {
1379     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
1380       @Override
1381       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1382           throws IOException {
1383         oserver.preScannerClose(ctx, s);
1384       }
1385     });
1386   }
1387 
1388   /**
1389    * @exception IOException Exception
1390    */
1391   public void postScannerClose(final InternalScanner s) throws IOException {
1392     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
1393       @Override
1394       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1395           throws IOException {
1396         oserver.postScannerClose(ctx, s);
1397       }
1398     });
1399   }
1400 
1401   /**
1402    * @param info
1403    * @param logKey
1404    * @param logEdit
1405    * @return true if default behavior should be bypassed, false otherwise
1406    * @throws IOException
1407    */
1408   public boolean preWALRestore(final HRegionInfo info, final HLogKey logKey,
1409       final WALEdit logEdit) throws IOException {
1410     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
1411       @Override
1412       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1413           throws IOException {
1414         oserver.preWALRestore(ctx, info, logKey, logEdit);
1415       }
1416     });
1417   }
1418 
1419   /**
1420    * @param info
1421    * @param logKey
1422    * @param logEdit
1423    * @throws IOException
1424    */
1425   public void postWALRestore(final HRegionInfo info, final HLogKey logKey, final WALEdit logEdit)
1426       throws IOException {
1427     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
1428       @Override
1429       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1430           throws IOException {
1431         oserver.postWALRestore(ctx, info, logKey, logEdit);
1432       }
1433     });
1434   }
1435 
1436   /**
1437    * @param familyPaths pairs of { CF, file path } submitted for bulk load
1438    * @return true if the default operation should be bypassed
1439    * @throws IOException
1440    */
1441   public boolean preBulkLoadHFile(final List<Pair<byte[], String>> familyPaths) throws IOException {
1442     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
1443       @Override
1444       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1445           throws IOException {
1446         oserver.preBulkLoadHFile(ctx, familyPaths);
1447       }
1448     });
1449   }
1450 
1451   /**
1452    * @param familyPaths pairs of { CF, file path } submitted for bulk load
1453    * @param hasLoaded whether load was successful or not
1454    * @return the possibly modified value of hasLoaded
1455    * @throws IOException
1456    */
1457   public boolean postBulkLoadHFile(final List<Pair<byte[], String>> familyPaths,
1458       boolean hasLoaded) throws IOException {
1459     return execOperationWithResult(hasLoaded,
1460         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
1461       @Override
1462       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1463           throws IOException {
1464         setResult(oserver.postBulkLoadHFile(ctx, familyPaths, getResult()));
1465       }
1466     });
1467   }
1468 
1469   public void postStartRegionOperation(final Operation op) throws IOException {
1470     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
1471       @Override
1472       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1473           throws IOException {
1474         oserver.postStartRegionOperation(ctx, op);
1475       }
1476     });
1477   }
1478 
1479   public void postCloseRegionOperation(final Operation op) throws IOException {
1480     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
1481       @Override
1482       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1483           throws IOException {
1484         oserver.postCloseRegionOperation(ctx, op);
1485       }
1486     });
1487   }
1488 
1489   /**
1490    * @param fs fileystem to read from
1491    * @param p path to the file
1492    * @param in {@link FSDataInputStreamWrapper}
1493    * @param size Full size of the file
1494    * @param cacheConf
1495    * @param r original reference file. This will be not null only when reading a split file.
1496    * @return a Reader instance to use instead of the base reader if overriding
1497    * default behavior, null otherwise
1498    * @throws IOException
1499    */
1500   public StoreFile.Reader preStoreFileReaderOpen(final FileSystem fs, final Path p,
1501       final FSDataInputStreamWrapper in, final long size, final CacheConfig cacheConf,
1502       final Reference r) throws IOException {
1503     return execOperationWithResult(null,
1504         coprocessors.isEmpty() ? null : new RegionOperationWithResult<StoreFile.Reader>() {
1505       @Override
1506       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1507           throws IOException {
1508         setResult(oserver.preStoreFileReaderOpen(ctx, fs, p, in, size, cacheConf, r, getResult()));
1509       }
1510     });
1511   }
1512 
1513   /**
1514    * @param fs fileystem to read from
1515    * @param p path to the file
1516    * @param in {@link FSDataInputStreamWrapper}
1517    * @param size Full size of the file
1518    * @param cacheConf
1519    * @param r original reference file. This will be not null only when reading a split file.
1520    * @param reader the base reader instance
1521    * @return The reader to use
1522    * @throws IOException
1523    */
1524   public StoreFile.Reader postStoreFileReaderOpen(final FileSystem fs, final Path p,
1525       final FSDataInputStreamWrapper in, final long size, final CacheConfig cacheConf,
1526       final Reference r, final StoreFile.Reader reader) throws IOException {
1527     return execOperationWithResult(reader,
1528         coprocessors.isEmpty() ? null : new RegionOperationWithResult<StoreFile.Reader>() {
1529       @Override
1530       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1531           throws IOException {
1532         setResult(oserver.postStoreFileReaderOpen(ctx, fs, p, in, size, cacheConf, r, getResult()));
1533       }
1534     });
1535   }
1536 
1537   public Cell postMutationBeforeWAL(final MutationType opType, final Mutation mutation,
1538       final Cell oldCell, Cell newCell) throws IOException {
1539     return execOperationWithResult(newCell,
1540         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Cell>() {
1541       @Override
1542       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1543           throws IOException {
1544         setResult(oserver.postMutationBeforeWAL(ctx, opType, mutation, oldCell, getResult()));
1545       }
1546     });
1547   }
1548 
1549   public Message preEndpointInvocation(final Service service, final String methodName,
1550       Message request) throws IOException {
1551     return execOperationWithResult(request,
1552         coprocessors.isEmpty() ? null : new EndpointOperationWithResult<Message>() {
1553       @Override
1554       public void call(EndpointObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1555           throws IOException {
1556         setResult(oserver.preEndpointInvocation(ctx, service, methodName, getResult()));
1557       }
1558     });
1559   }
1560 
1561   public void postEndpointInvocation(final Service service, final String methodName,
1562       final Message request, final Message.Builder responseBuilder) throws IOException {
1563     execOperation(coprocessors.isEmpty() ? null : new EndpointOperation() {
1564       @Override
1565       public void call(EndpointObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1566           throws IOException {
1567         oserver.postEndpointInvocation(ctx, service, methodName, request, responseBuilder);
1568       }
1569     });
1570   }
1571 
1572   public DeleteTracker postInstantiateDeleteTracker(DeleteTracker tracker) throws IOException {
1573     return execOperationWithResult(tracker,
1574         coprocessors.isEmpty() ? null : new RegionOperationWithResult<DeleteTracker>() {
1575       @Override
1576       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1577           throws IOException {
1578         setResult(oserver.postInstantiateDeleteTracker(ctx, getResult()));
1579       }
1580     });
1581   }
1582 
1583   public Map<String, DescriptiveStatistics> getCoprocessorExecutionStatistics() {
1584     Map<String, DescriptiveStatistics> results = new HashMap<String, DescriptiveStatistics>();
1585     for (RegionEnvironment env : coprocessors) {
1586       DescriptiveStatistics ds = new DescriptiveStatistics();
1587       if (env.getInstance() instanceof RegionObserver) {
1588         for (Long time : env.getExecutionLatenciesNanos()) {
1589           ds.addValue(time);
1590         }
1591         // Ensures that web ui circumvents the display of NaN values when there are zero samples.
1592         if (ds.getN() == 0) {
1593           ds.addValue(0);
1594         }
1595         results.put(env.getInstance().getClass().getSimpleName(), ds);
1596       }
1597     }
1598     return results;
1599   }
1600 
1601   private static abstract class CoprocessorOperation
1602       extends ObserverContext<RegionCoprocessorEnvironment> {
1603     public abstract void call(Coprocessor observer,
1604         ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException;
1605     public abstract boolean hasCall(Coprocessor observer);
1606     public void postEnvCall(RegionEnvironment env) { }
1607   }
1608 
1609   private static abstract class RegionOperation extends CoprocessorOperation {
1610     public abstract void call(RegionObserver observer,
1611         ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException;
1612 
1613     public boolean hasCall(Coprocessor observer) {
1614       return observer instanceof RegionObserver;
1615     }
1616 
1617     public void call(Coprocessor observer, ObserverContext<RegionCoprocessorEnvironment> ctx)
1618         throws IOException {
1619       call((RegionObserver)observer, ctx);
1620     }
1621   }
1622 
1623   private static abstract class RegionOperationWithResult<T> extends RegionOperation {
1624     private T result = null;
1625     public void setResult(final T result) { this.result = result; }
1626     public T getResult() { return this.result; }
1627   }
1628 
1629   private static abstract class EndpointOperation extends CoprocessorOperation {
1630     public abstract void call(EndpointObserver observer,
1631         ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException;
1632 
1633     public boolean hasCall(Coprocessor observer) {
1634       return observer instanceof EndpointObserver;
1635     }
1636 
1637     public void call(Coprocessor observer, ObserverContext<RegionCoprocessorEnvironment> ctx)
1638         throws IOException {
1639       call((EndpointObserver)observer, ctx);
1640     }
1641   }
1642 
1643   private static abstract class EndpointOperationWithResult<T> extends EndpointOperation {
1644     private T result = null;
1645     public void setResult(final T result) { this.result = result; }
1646     public T getResult() { return this.result; }
1647   }
1648 
1649   private boolean execOperation(final CoprocessorOperation ctx)
1650       throws IOException {
1651     return execOperation(true, ctx);
1652   }
1653 
1654   private <T> T execOperationWithResult(final T defaultValue,
1655       final RegionOperationWithResult<T> ctx) throws IOException {
1656     if (ctx == null) return defaultValue;
1657     ctx.setResult(defaultValue);
1658     execOperation(true, ctx);
1659     return ctx.getResult();
1660   }
1661 
1662   private <T> T execOperationWithResult(final boolean ifBypass, final T defaultValue,
1663       final RegionOperationWithResult<T> ctx) throws IOException {
1664     boolean bypass = false;
1665     T result = defaultValue;
1666     if (ctx != null) {
1667       ctx.setResult(defaultValue);
1668       bypass = execOperation(true, ctx);
1669       result = ctx.getResult();
1670     }
1671     return bypass == ifBypass ? result : null;
1672   }
1673 
1674   private <T> T execOperationWithResult(final T defaultValue,
1675       final EndpointOperationWithResult<T> ctx) throws IOException {
1676     if (ctx == null) return defaultValue;
1677     ctx.setResult(defaultValue);
1678     execOperation(true, ctx);
1679     return ctx.getResult();
1680   }
1681 
1682   private boolean execOperation(final boolean earlyExit, final CoprocessorOperation ctx)
1683       throws IOException {
1684     boolean bypass = false;
1685     for (RegionEnvironment env: coprocessors) {
1686       Coprocessor observer = env.getInstance();
1687       if (ctx.hasCall(observer)) {
1688         long startTime = System.nanoTime();
1689         ctx.prepare(env);
1690         Thread currentThread = Thread.currentThread();
1691         ClassLoader cl = currentThread.getContextClassLoader();
1692         try {
1693           currentThread.setContextClassLoader(env.getClassLoader());
1694           ctx.call(observer, ctx);
1695         } catch (Throwable e) {
1696           handleCoprocessorThrowable(env, e);
1697         } finally {
1698           currentThread.setContextClassLoader(cl);
1699         }
1700         env.offerExecutionLatency(System.nanoTime() - startTime);
1701         bypass |= ctx.shouldBypass();
1702         if (earlyExit && ctx.shouldComplete()) {
1703           break;
1704         }
1705       }
1706 
1707       ctx.postEnvCall(env);
1708     }
1709     return bypass;
1710   }
1711 }