1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.client;
19
20 import java.io.IOException;
21 import java.util.LinkedList;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.hbase.classification.InterfaceAudience;
26 import org.apache.hadoop.hbase.classification.InterfaceStability;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.hbase.Cell;
29 import org.apache.hadoop.hbase.DoNotRetryIOException;
30 import org.apache.hadoop.hbase.HBaseConfiguration;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.HRegionInfo;
33 import org.apache.hadoop.hbase.KeyValueUtil;
34 import org.apache.hadoop.hbase.NotServingRegionException;
35 import org.apache.hadoop.hbase.TableName;
36 import org.apache.hadoop.hbase.UnknownScannerException;
37 import org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException;
38 import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
39 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
40 import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
41 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
42 import org.apache.hadoop.hbase.util.Bytes;
43
44
45
46
47
48
49 @InterfaceAudience.Public
50 @InterfaceStability.Stable
51 public class ClientScanner extends AbstractClientScanner {
52 private final Log LOG = LogFactory.getLog(this.getClass());
53 protected Scan scan;
54 protected boolean closed = false;
55
56
57 protected HRegionInfo currentRegion = null;
58 protected ScannerCallable callable = null;
59 protected final LinkedList<Result> cache = new LinkedList<Result>();
60 protected final int caching;
61 protected long lastNext;
62
63 protected Result lastResult = null;
64 protected final long maxScannerResultSize;
65 private final HConnection connection;
66 private final TableName tableName;
67 protected final int scannerTimeout;
68 protected boolean scanMetricsPublished = false;
69 protected RpcRetryingCaller<Result []> caller;
70 protected RpcControllerFactory rpcControllerFactory;
71
72
73
74
75
76
77
78
79
80
81
82 @Deprecated
83 public ClientScanner(final Configuration conf, final Scan scan,
84 final TableName tableName) throws IOException {
85 this(conf, scan, tableName, HConnectionManager.getConnection(conf));
86 }
87
88
89
90
91 @Deprecated
92 public ClientScanner(final Configuration conf, final Scan scan,
93 final byte [] tableName) throws IOException {
94 this(conf, scan, TableName.valueOf(tableName));
95 }
96
97
98
99
100
101
102
103
104
105
106
107
108 public ClientScanner(final Configuration conf, final Scan scan, final TableName tableName,
109 HConnection connection) throws IOException {
110 this(conf, scan, tableName, connection,
111 RpcRetryingCallerFactory.instantiate(conf, connection.getStatisticsTracker()),
112 RpcControllerFactory.instantiate(conf));
113 }
114
115
116
117
118 @Deprecated
119 public ClientScanner(final Configuration conf, final Scan scan, final byte [] tableName,
120 HConnection connection) throws IOException {
121 this(conf, scan, TableName.valueOf(tableName), connection, new RpcRetryingCallerFactory(conf),
122 RpcControllerFactory.instantiate(conf));
123 }
124
125
126
127
128
129
130
131 @Deprecated
132 public ClientScanner(final Configuration conf, final Scan scan, final TableName tableName,
133 HConnection connection, RpcRetryingCallerFactory rpcFactory) throws IOException {
134 this(conf, scan, tableName, connection, rpcFactory, RpcControllerFactory.instantiate(conf));
135 }
136
137
138
139
140
141
142
143
144
145
146 public ClientScanner(final Configuration conf, final Scan scan, final TableName tableName,
147 HConnection connection, RpcRetryingCallerFactory rpcFactory,
148 RpcControllerFactory controllerFactory) throws IOException {
149 if (LOG.isTraceEnabled()) {
150 LOG.trace("Scan table=" + tableName
151 + ", startRow=" + Bytes.toStringBinary(scan.getStartRow()));
152 }
153 this.scan = scan;
154 this.tableName = tableName;
155 this.lastNext = System.currentTimeMillis();
156 this.connection = connection;
157 if (scan.getMaxResultSize() > 0) {
158 this.maxScannerResultSize = scan.getMaxResultSize();
159 } else {
160 this.maxScannerResultSize = conf.getLong(
161 HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
162 HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
163 }
164 this.scannerTimeout = HBaseConfiguration.getInt(conf,
165 HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
166 HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
167 HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
168
169
170 initScanMetrics(scan);
171
172
173 if (this.scan.getCaching() > 0) {
174 this.caching = this.scan.getCaching();
175 } else {
176 this.caching = conf.getInt(
177 HConstants.HBASE_CLIENT_SCANNER_CACHING,
178 HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING);
179 }
180
181 this.caller = rpcFactory.<Result[]> newCaller();
182 this.rpcControllerFactory = controllerFactory;
183
184 initializeScannerInConstruction();
185 }
186
187 protected void initializeScannerInConstruction() throws IOException{
188
189 nextScanner(this.caching, false);
190 }
191
192 protected HConnection getConnection() {
193 return this.connection;
194 }
195
196
197
198
199
200 @Deprecated
201 protected byte [] getTableName() {
202 return this.tableName.getName();
203 }
204
205 protected TableName getTable() {
206 return this.tableName;
207 }
208
209 protected Scan getScan() {
210 return scan;
211 }
212
213 protected long getTimestamp() {
214 return lastNext;
215 }
216
217
218 protected boolean checkScanStopRow(final byte [] endKey) {
219 if (this.scan.getStopRow().length > 0) {
220
221 byte [] stopRow = scan.getStopRow();
222 int cmp = Bytes.compareTo(stopRow, 0, stopRow.length,
223 endKey, 0, endKey.length);
224 if (cmp <= 0) {
225
226
227 return true;
228 }
229 }
230 return false;
231 }
232
233
234
235
236
237
238
239
240
241
242 protected boolean nextScanner(int nbRows, final boolean done)
243 throws IOException {
244
245 if (this.callable != null) {
246 this.callable.setClose();
247 this.caller.callWithRetries(callable);
248 this.callable = null;
249 }
250
251
252 byte [] localStartKey;
253
254
255 if (this.currentRegion != null) {
256 byte [] endKey = this.currentRegion.getEndKey();
257 if (endKey == null ||
258 Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY) ||
259 checkScanStopRow(endKey) ||
260 done) {
261 close();
262 if (LOG.isTraceEnabled()) {
263 LOG.trace("Finished " + this.currentRegion);
264 }
265 return false;
266 }
267 localStartKey = endKey;
268 if (LOG.isTraceEnabled()) {
269 LOG.trace("Finished " + this.currentRegion);
270 }
271 } else {
272 localStartKey = this.scan.getStartRow();
273 }
274
275 if (LOG.isDebugEnabled() && this.currentRegion != null) {
276
277 LOG.debug("Advancing internal scanner to startKey at '" +
278 Bytes.toStringBinary(localStartKey) + "'");
279 }
280 try {
281 callable = getScannerCallable(localStartKey, nbRows);
282
283
284 this.caller.callWithRetries(callable);
285 this.currentRegion = callable.getHRegionInfo();
286 if (this.scanMetrics != null) {
287 this.scanMetrics.countOfRegions.incrementAndGet();
288 }
289 } catch (IOException e) {
290 close();
291 throw e;
292 }
293 return true;
294 }
295
296 @InterfaceAudience.Private
297 protected ScannerCallable getScannerCallable(byte [] localStartKey,
298 int nbRows) {
299 scan.setStartRow(localStartKey);
300 ScannerCallable s = new ScannerCallable(getConnection(),
301 getTable(), scan, this.scanMetrics, rpcControllerFactory.newController());
302 s.setCaching(nbRows);
303 return s;
304 }
305
306
307
308
309
310
311
312
313
314
315
316
317 protected void writeScanMetrics() {
318 if (this.scanMetrics == null || scanMetricsPublished) {
319 return;
320 }
321 MapReduceProtos.ScanMetrics pScanMetrics = ProtobufUtil.toScanMetrics(scanMetrics);
322 scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA, pScanMetrics.toByteArray());
323 scanMetricsPublished = true;
324 }
325
326 @Override
327 public Result next() throws IOException {
328
329 if (cache.size() == 0 && this.closed) {
330 return null;
331 }
332 if (cache.size() == 0) {
333 Result [] values = null;
334 long remainingResultSize = maxScannerResultSize;
335 int countdown = this.caching;
336
337
338 callable.setCaching(this.caching);
339
340
341 boolean skipFirst = false;
342 boolean retryAfterOutOfOrderException = true;
343 do {
344 try {
345 if (skipFirst) {
346
347
348 callable.setCaching(1);
349 values = this.caller.callWithRetries(callable);
350 callable.setCaching(this.caching);
351 skipFirst = false;
352 }
353
354
355
356 values = this.caller.callWithRetries(callable);
357 if (skipFirst && values != null && values.length == 1) {
358 skipFirst = false;
359 values = this.caller.callWithRetries(callable);
360 }
361 retryAfterOutOfOrderException = true;
362 } catch (DoNotRetryIOException e) {
363
364
365 if (e instanceof UnknownScannerException) {
366 long timeout = lastNext + scannerTimeout;
367
368
369
370 if (timeout < System.currentTimeMillis()) {
371 long elapsed = System.currentTimeMillis() - lastNext;
372 ScannerTimeoutException ex = new ScannerTimeoutException(
373 elapsed + "ms passed since the last invocation, " +
374 "timeout is currently set to " + scannerTimeout);
375 ex.initCause(e);
376 throw ex;
377 }
378 } else {
379
380
381 Throwable cause = e.getCause();
382 if ((cause != null && cause instanceof NotServingRegionException) ||
383 (cause != null && cause instanceof RegionServerStoppedException) ||
384 e instanceof OutOfOrderScannerNextException) {
385
386
387
388 } else {
389 throw e;
390 }
391 }
392
393 if (this.lastResult != null) {
394
395
396
397
398
399
400 this.scan.setStartRow(this.lastResult.getRow());
401
402
403
404 skipFirst = true;
405 }
406 if (e instanceof OutOfOrderScannerNextException) {
407 if (retryAfterOutOfOrderException) {
408 retryAfterOutOfOrderException = false;
409 } else {
410
411 throw new DoNotRetryIOException("Failed after retry of " +
412 "OutOfOrderScannerNextException: was there a rpc timeout?", e);
413 }
414 }
415
416 this.currentRegion = null;
417
418
419 callable = null;
420
421 continue;
422 }
423 long currentTime = System.currentTimeMillis();
424 if (this.scanMetrics != null ) {
425 this.scanMetrics.sumOfMillisSecBetweenNexts.addAndGet(currentTime-lastNext);
426 }
427 lastNext = currentTime;
428 if (values != null && values.length > 0) {
429 for (Result rs : values) {
430 cache.add(rs);
431 for (Cell kv : rs.rawCells()) {
432
433 remainingResultSize -= KeyValueUtil.ensureKeyValue(kv).heapSize();
434 }
435 countdown--;
436 this.lastResult = rs;
437 }
438 }
439
440 } while (remainingResultSize > 0 && countdown > 0 && nextScanner(countdown, values == null));
441 }
442
443 if (cache.size() > 0) {
444 return cache.poll();
445 }
446
447
448 writeScanMetrics();
449 return null;
450 }
451
452 @Override
453 public void close() {
454 if (!scanMetricsPublished) writeScanMetrics();
455 if (callable != null) {
456 callable.setClose();
457 try {
458 this.caller.callWithRetries(callable);
459 } catch (UnknownScannerException e) {
460
461
462
463 } catch (IOException e) {
464
465 LOG.warn("scanner failed to close. Exception follows: " + e);
466 }
467 callable = null;
468 }
469 closed = true;
470 }
471 }