Repository: asterixdb Updated Branches: refs/heads/master ecb045e50 -> d195de144
Fix for ASTERIXDB-1738: Change feed fails to delete record with meta PK Change-Id: I5ce8d9c69f96593ee305cfad4b44e486c9f1d6cc Reviewed-on: https://asterix-gerrit.ics.uci.edu/1351 Reviewed-by: abdullah alamoudi <bamou...@gmail.com> Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> BAD: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/d195de14 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/d195de14 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/d195de14 Branch: refs/heads/master Commit: d195de1443b0ccf8222770452cced8bcb5c25d5f Parents: ecb045e Author: Ildar Absalyamov <ildar.absalya...@gmail.com> Authored: Tue Dec 6 23:09:20 2016 -0800 Committer: Ildar Absalyamov <ildar.absalya...@gmail.com> Committed: Mon Feb 6 23:25:33 2017 -0800 ---------------------------------------------------------------------- asterixdb/asterix-app/pom.xml | 3 +- .../change-feed-with-meta-csv.1.ddl.aql | 61 ++++++++++++++++++++ .../change-feed-with-meta-csv.2.update.aql | 27 +++++++++ .../change-feed-with-meta-csv.3.query.aql | 30 ++++++++++ .../change-feed-with-meta-csv.4.ddl.aql | 24 ++++++++ .../change-feed-with-meta-csv.3.adm | 2 + .../src/test/resources/runtimets/testsuite.xml | 5 ++ .../external/input/record/CharArrayRecord.java | 2 +- .../src/test/resources/change_feed.csv | 5 ++ 9 files changed, 157 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d195de14/asterixdb/asterix-app/pom.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml index 597e328..99e58df 100644 --- a/asterixdb/asterix-app/pom.xml +++ b/asterixdb/asterix-app/pom.xml @@ -101,7 +101,7 @@ </configuration> </execution> <execution> - <id>copy-beer-csv</id> + <id>copy-external-data-resources</id> <phase>generate-resources</phase> <goals> <goal>copy-resources</goal> @@ -114,6 +114,7 @@ <directory>../asterix-external-data/src/test/resources</directory> <includes> <include>beer.csv</include> + <include>change_feed.csv</include> </includes> </resource> </resources> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d195de14/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.1.ddl.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.1.ddl.aql new file mode 100644 index 0000000..b695a5d --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.1.ddl.aql @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Create a dataset with meta part and attempt to load the csv file using localFS change feed + * Expected Res : Success + * Date : 19th Nov 2017 + */ + drop dataverse test if exists; + create dataverse test; + use dataverse test; + + create type TestType as closed { + test: string + } + create type MetaType as { + id:int + } + + create dataset TestDS(TestType) with meta(MetaType) primary key meta().id; + create index TestIdx on TestDS(test); + drop feed TestFeed if exists; + create feed TestFeed using localfs + ( + ("parser"="record-with-metadata"), + ("reader"="localfs"), + ("path"="asterix_nc1://target/data/csv/change_feed.csv"), + //type of the record + ("type-name"="TestType"), + //type of the meta record + ("meta-type-name"="MetaType"), + //format of the record + ("record-format"="adm"), + // format of the meta record + ("format"="csv"), + ("delimiter"=","), + // index of the primary key the record (meta or orig) + ("key-indexes"="0"), + // indicates that PK originates from meta + ("key-indicators"="1"), + //index of the record in the meta record + ("record-index"="1"), + ("change-feed"="true"), + // whether CSV header is present in the input + ("header"="false") + ); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d195de14/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.2.update.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.2.update.aql new file mode 100644 index 0000000..c83a6c0 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.2.update.aql @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Create a dataset with meta part and attempt to load the csv file using localFS change feed + * Expected Res : Success + * Date : 19th Nov 2017 + */ + use dataverse test; + + set wait-for-completion-feed "true"; + connect feed TestFeed to dataset TestDS; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d195de14/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.3.query.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.3.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.3.query.aql new file mode 100644 index 0000000..317ab90 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.3.query.aql @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Create a dataset with meta part and attempt to load the csv file using localFS change feed + * Expected Res : Success + * Date : 19th Nov 2017 + */ + use dataverse test; + + for $x in dataset TestDS + return { + "record": $x, + "id": meta().id + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d195de14/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.4.ddl.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.4.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.4.ddl.aql new file mode 100644 index 0000000..21697d6 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.4.ddl.aql @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Create a dataset with meta part and attempt to load the csv file using localFS change feed + * Expected Res : Success + * Date : 19th Nov 2017 + */ + drop dataverse test; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d195de14/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.3.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.3.adm new file mode 100644 index 0000000..e083535 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-csv/change-feed-with-meta-csv.3.adm @@ -0,0 +1,2 @@ +{ "record": { "test": "test1" }, "id": 1 } +{ "record": { "test": "test4" }, "id": 2 } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d195de14/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml index 5be8639..e4af727 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml @@ -138,6 +138,11 @@ </compilation-unit> </test-case> <test-case FilePath="feeds"> + <compilation-unit name="change-feed-with-meta-csv"> + <output-dir compare="Text">change-feed-with-meta-csv</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="feeds"> <compilation-unit name="drop-nonexistent-feed"> <output-dir compare="Text">drop-nonexistent-feed</output-dir> </compilation-unit> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d195de14/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java index ff0cf84..aff00fc 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java @@ -94,7 +94,7 @@ public class CharArrayRecord implements IRawRecord<char[]> { } public void endRecord() throws IOException { - if (value[size - 1] != ExternalDataConstants.LF) { + if (size > 0 && value[size - 1] != ExternalDataConstants.LF) { appendChar(ExternalDataConstants.LF); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d195de14/asterixdb/asterix-external-data/src/test/resources/change_feed.csv ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/test/resources/change_feed.csv b/asterixdb/asterix-external-data/src/test/resources/change_feed.csv new file mode 100644 index 0000000..944a0f3 --- /dev/null +++ b/asterixdb/asterix-external-data/src/test/resources/change_feed.csv @@ -0,0 +1,5 @@ +1,"{""test"":""test1""}" +2,"{""test"":""test2""}" +3,"{""test"":""test3""}" +3, +2,"{""test"":""test4""}" \ No newline at end of file
