Wenhai Li has uploaded a new change for review. https://asterix-gerrit.ics.uci.edu/1077
Change subject: Patch for the remaning issue inconsistent with UTF8. ...................................................................... Patch for the remaning issue inconsistent with UTF8. commit 02bed994f57c6cccc026a058a1d0e9b1e6320bc9 Author: Michael <[email protected]> Date: Sun Aug 14 04:08:46 2016 -0700 Fix the issure-1478. commit 098e7e55806d372e60c018f5c451a66871f3cde4 Author: Michael <[email protected]> Date: Fri Aug 12 08:04:21 2016 -0700 Add a testCases for issue-1478 The following commits from your working branch will be included: commit 59f33057a840cb67d2cfef5a8e730a515dca7322 Merge: a253268 bd3a535 Author: Michael <[email protected]> Date: Fri Aug 12 07:46:47 2016 -0700 Merge branch 'master' into chinese commit a253268e9ded282a8508c4d977b5dab6351408f2 Author: Michael <[email protected]> Date: Fri Aug 12 07:43:45 2016 -0700 Add the utf8 testCases. Change-Id: Idb302dc604fcd71811de550d3d4bd727c81a13ee --- A asterixdb/asterix-app/data/adm-load/utf8.adm A asterixdb/asterix-app/src/test/resources/runtimets/queries/load/utf8/utf8.1.ddl.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/load/utf8/utf8.2.update.aql A asterixdb/asterix-app/src/test/resources/runtimets/queries/load/utf8/utf8.3.query.aql A asterixdb/asterix-app/src/test/resources/runtimets/results/load/utf8/utf8.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java 7 files changed, 197 insertions(+), 1 deletion(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/77/1077/1 diff --git a/asterixdb/asterix-app/data/adm-load/utf8.adm b/asterixdb/asterix-app/data/adm-load/utf8.adm new file mode 100644 index 0000000..2621bc4 --- /dev/null +++ b/asterixdb/asterix-app/data/adm-load/utf8.adm @@ -0,0 +1,100 @@ +{"id":"1","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"2","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"3","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"4","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"5","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"6","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"7","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"8","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"9","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"10","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"11","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"12","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"13","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"14","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"15","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"16","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"17","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"18","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"19","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"20","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"21","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"22","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"23","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"24","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"25","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"26","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"27","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"28","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"29","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"30","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"31","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"32","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"33","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"34","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"35","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"36","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"37","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"38","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"39","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"40","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"41","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"42","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"43","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"44","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"45","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"46","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"47","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"48","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"49","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"50","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"51","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"52","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"53","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"54","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"55","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"56","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"57","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"58","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"59","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"60","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"61","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"62","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"63","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"64","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"65","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"66","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"67","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"68","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"69","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"70","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"71","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"72","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"73","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"74","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"75","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"76","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"77","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"78","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"79","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"80","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"81","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"82","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"83","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"84","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"85","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"86","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"87","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"88","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"89","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"90","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"91","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"92","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"93","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"94","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"95","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"96","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"97","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"98","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"99","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} +{"id":"100","description":"随着人们信用活动的繁荣、社会对信用服务需求的激增，构建一个完整"} diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/load/utf8/utf8.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/load/utf8/utf8.1.ddl.aql new file mode 100644 index 0000000..bcd3d46 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/load/utf8/utf8.1.ddl.aql @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +drop dataverse test if exists; +create dataverse test +use dataverse test; + +create type DocType as open { + id: string, + description: string? +}; + +create dataset Doc (DocType) +primary key id; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/load/utf8/utf8.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/load/utf8/utf8.2.update.aql new file mode 100644 index 0000000..4d4f4e5 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/load/utf8/utf8.2.update.aql @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * + * CSV file loading utf8 + * Expected result: success + * + */ + +use dataverse test; + +load dataset Doc +using localfs +(("path"="asterix_nc1://data/adm-load/utf8.adm"),("format"="adm")); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/load/utf8/utf8.3.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/load/utf8/utf8.3.query.aql new file mode 100644 index 0000000..95507bf --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/load/utf8/utf8.3.query.aql @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use dataverse test; + +let $s := count( +for $i in dataset Doc +return $i) +return $s diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/load/utf8/utf8.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/load/utf8/utf8.1.adm new file mode 100644 index 0000000..29d6383 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/load/utf8/utf8.1.adm @@ -0,0 +1 @@ +100 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml index 862ca26..5ee2bb5 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml @@ -6555,6 +6555,11 @@ <output-dir compare="Text">adm_binary</output-dir> </compilation-unit> </test-case> + <test-case FilePath="load"> + <compilation-unit name="utf8"> + <output-dir compare="Text">utf8</output-dir> + </compilation-unit> + </test-case> </test-group> <test-group name="hints"> <test-case FilePath="hints"> diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java index 94333d1..8e166c0 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java @@ -38,6 +38,7 @@ private CharBuffer charBuffer = CharBuffer.allocate(ExternalDataConstants.DEFAULT_BUFFER_SIZE); private CharsetDecoder decoder; private boolean done = false; + private boolean remaining = false; public AsterixInputStreamReader(AsterixInputStream in) { this.in = in; @@ -75,6 +76,7 @@ charBuffer.clear(); while (charBuffer.position() == 0) { if (byteBuffer.hasRemaining()) { + remaining = true; decoder.decode(byteBuffer, charBuffer, false); System.arraycopy(charBuffer.array(), 0, cbuf, offset, charBuffer.position()); if (charBuffer.position() > 0) { @@ -97,8 +99,13 @@ done = true; return len; } - byteBuffer.position(len); + if (remaining) { + byteBuffer.position(len + byteBuffer.position()); + } else { + byteBuffer.position(len); + } byteBuffer.flip(); + remaining = false; decoder.decode(byteBuffer, charBuffer, false); System.arraycopy(charBuffer.array(), 0, cbuf, offset, charBuffer.position()); } -- To view, visit https://asterix-gerrit.ics.uci.edu/1077 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Idb302dc604fcd71811de550d3d4bd727c81a13ee Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Wenhai Li <[email protected]>
