Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/720
to look at the new patch set (#9).
Change subject: Improve Error Handling in Local Directory Feeds
......................................................................
Improve Error Handling in Local Directory Feeds
This change improves handling of two error types for filesystem
based feeds. The first one is the handling of IO Errors which
causes the input stream to be closed, and the second one is
reacting to missed filesystem events. In both cases, we scan the
directory and compare it with the history we have in order to
resume from where we last left off.
In addition, this change includes some refactoring in external
data. Particularly, we get rid of the stream provider layer and
instead, stream factories create input streams directly. This
is consistent with record reader factories which create readers
directly without reader providers.
Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
---
M .gitattributes
M .gitignore
M
asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
M
asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.2.update.aql
M
asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.3.query.aql
A
asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
R
asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
R
asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
D
asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
R
asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
A
asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
R
asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
A
asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
A
asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
R
asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
R
asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
R
asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
R
asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
D
asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
D
asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
D
asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
D
asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
D
asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
M
asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
M
asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
M
asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
59 files changed, 930 insertions(+), 803 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/20/720/9
--
To view, visit https://asterix-gerrit.ics.uci.edu/720
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <[email protected]>
Gerrit-Reviewer: Jenkins <[email protected]>
Gerrit-Reviewer: Murtadha Hubail <[email protected]>
Gerrit-Reviewer: Till Westmann <[email protected]>
Gerrit-Reviewer: abdullah alamoudi <[email protected]>
