Xikui Wang has uploaded a new change for review. https://asterix-gerrit.ics.uci.edu/3026
Change subject: [ASTERIXDB-2480][ING] Fix http-feed compatibility with JSON parser ...................................................................... [ASTERIXDB-2480][ING] Fix http-feed compatibility with JSON parser - user model changes: no - storage format changes: no - interface changes: no Details: The generic record used in current http-feed returns length of "-1" which will cause the JSON parser to fail. For compatibility consideration, we should use CharArrayRecord instead. Change-Id: I719c98376583a3316dbc291696df65a97dd4b26e --- A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.3.post.http A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.4.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.5.pollquery.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.6.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.1.adm A asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.2.adm M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java 10 files changed, 154 insertions(+), 2 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/26/3026/1 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.1.ddl.sqlpp new file mode 100644 index 0000000..246f7e1 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.1.ddl.sqlpp @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +drop dataverse test if exists; +create dataverse test; +use test; + +create type WorrisomeNotificationType as { + dataverseName: string, + channelName: string +}; +create feed WorrisomeTweetsFeed with { + "adapter-name" : "http_adapter", + "addresses" : "asterix_nc1:10001", + "address-type" : "NC", + "type-name" : "WorrisomeNotificationType", + "format" : "json" +}; + +create dataset WorrisomeNotifications(WorrisomeNotificationType) primary key dataverseName; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.2.update.sqlpp new file mode 100644 index 0000000..cc180a3 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.2.update.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; +set `wait-for-completion-feed` "false"; + +connect feed WorrisomeTweetsFeed to dataset WorrisomeNotifications; +start feed WorrisomeTweetsFeed; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.3.post.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.3.post.http new file mode 100644 index 0000000..350e6bf --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.3.post.http @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +nc:asterix_nc1:10001 / +--body={ "dataverseName":"dhs", "channelName":"WorrisomeTweetsNearMonumentsChannel", "channelExecutionTime":"2018-11-12T18:43:48.508Z"} \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.4.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.4.update.sqlpp new file mode 100644 index 0000000..90c3f8e --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.4.update.sqlpp @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; +stop feed WorrisomeTweetsFeed; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.5.pollquery.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.5.pollquery.sqlpp new file mode 100644 index 0000000..996542b --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.5.pollquery.sqlpp @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +// polltimeoutsecs=5 + +use test; + +select value count(t) from WorrisomeNotifications as t; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.6.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.6.ddl.sqlpp new file mode 100644 index 0000000..c17ef97 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/http_feed_json/http_feed_json.6.ddl.sqlpp @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; +drop dataverse test; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.1.adm new file mode 100644 index 0000000..56a6051 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.1.adm @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.2.adm new file mode 100644 index 0000000..56a6051 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/http_feed_json/http_feed.2.adm @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index e52449c..72a96a6 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -9984,6 +9984,11 @@ <output-dir compare="Text">http_feed</output-dir> </compilation-unit> </test-case> + <test-case FilePath="feeds"> + <compilation-unit name="http_feed_json"> + <output-dir compare="Text">http_feed_json</output-dir> + </compilation-unit> + </test-case> </test-group> <test-group name="meta"> <test-case FilePath="meta"> diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java index 2979641..a212f49 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReader.java @@ -27,6 +27,7 @@ import org.apache.asterix.external.api.IRawRecord; import org.apache.asterix.external.api.IRecordReader; import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController; +import org.apache.asterix.external.input.record.CharArrayRecord; import org.apache.asterix.external.input.record.GenericRecord; import org.apache.asterix.external.util.FeedLogManager; import org.apache.hyracks.http.api.IServletRequest; @@ -49,7 +50,7 @@ private static final String DEFAULT_ENTRY_POINT = "/"; private static final int DEFAULT_QUEUE_SIZE = 128; private LinkedBlockingQueue<String> inputQ; - private GenericRecord<char[]> record; + private CharArrayRecord record; private boolean closed = false; private WebManager webManager; private HttpServer webServer; @@ -57,7 +58,7 @@ public HttpServerRecordReader(int port, String entryPoint, int queueSize, HttpServerConfig httpServerConfig) throws Exception { this.inputQ = new LinkedBlockingQueue<>(queueSize > 0 ? queueSize : DEFAULT_QUEUE_SIZE); - this.record = new GenericRecord<>(); + this.record = new CharArrayRecord(); webManager = new WebManager(); webServer = new HttpServer(webManager.getBosses(), webManager.getWorkers(), port, httpServerConfig); webServer.addServlet(new HttpFeedServlet(webServer.ctx(), -- To view, visit https://asterix-gerrit.ics.uci.edu/3026 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I719c98376583a3316dbc291696df65a97dd4b26e Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang <[email protected]>
