Repository: carbondata Updated Branches: refs/heads/master 71d617955 -> 019f5cd06
[CARBONDATA-2981] Support read primitive data type in CSDK 1.support readNextCarbonRow 2.support read different primitive data type in c code from java side: int double short long string 3.support some data type and convert: date timestamp varchar decimal array<T> 4.remove readNextStringRow This closes #2792 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/019f5cd0 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/019f5cd0 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/019f5cd0 Branch: refs/heads/master Commit: 019f5cd06f09b7006507b6f8a912b4f0a2661357 Parents: 71d6179 Author: xubo245 <[email protected]> Authored: Tue Oct 9 17:58:48 2018 +0800 Committer: kunal642 <[email protected]> Committed: Fri Oct 26 18:54:59 2018 +0530 ---------------------------------------------------------------------- README.md | 1 + .../core/datastore/row/CarbonRow.java | 1 + docs/CSDK-guide.md | 197 -------------- docs/csdk-guide.md | 103 +++++++ docs/documentation.md | 2 +- store/CSDK/CMakeLists.txt | 22 +- store/CSDK/CarbonReader.cpp | 103 ------- store/CSDK/CarbonReader.h | 105 -------- store/CSDK/main.cpp | 200 -------------- store/CSDK/src/CarbonReader.cpp | 176 ++++++++++++ store/CSDK/src/CarbonReader.h | 112 ++++++++ store/CSDK/src/CarbonRow.cpp | 193 ++++++++++++++ store/CSDK/src/CarbonRow.h | 153 +++++++++++ store/CSDK/test/main.cpp | 266 +++++++++++++++++++ .../carbondata/sdk/file/CarbonReader.java | 30 --- .../org/apache/carbondata/sdk/file/Field.java | 9 + .../org/apache/carbondata/sdk/file/RowUtil.java | 146 ++++++++++ .../sdk/file/AvroCarbonWriterTest.java | 3 +- .../carbondata/sdk/file/CarbonReaderTest.java | 208 +++++++++++++++ 19 files changed, 1382 insertions(+), 648 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index 87bb71e..d472e13 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ CarbonData is built using Apache Maven, to [build CarbonData](https://github.com * [CarbonData Pre-aggregate DataMap](https://github.com/apache/carbondata/blob/master/docs/preaggregate-datamap-guide.md) * [CarbonData Timeseries DataMap](https://github.com/apache/carbondata/blob/master/docs/timeseries-datamap-guide.md) * [SDK Guide](https://github.com/apache/carbondata/blob/master/docs/sdk-guide.md) +* [CSDK Guide](https://github.com/apache/carbondata/blob/master/docs/csdk-guide.md) * [Performance Tuning](https://github.com/apache/carbondata/blob/master/docs/performance-tuning.md) * [S3 Storage](https://github.com/apache/carbondata/blob/master/docs/s3-guide.md) * [Carbon as Spark's Datasource](https://github.com/apache/carbondata/blob/master/docs/carbon-as-spark-datasource-guide.md) http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java b/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java index 48775d4..1141707 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java @@ -72,6 +72,7 @@ public class CarbonRow implements Serializable { public Object[] getRawData() { return rawData; } + public void setRawData(Object[] rawData) { this.rawData = rawData; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/docs/CSDK-guide.md ---------------------------------------------------------------------- diff --git a/docs/CSDK-guide.md b/docs/CSDK-guide.md deleted file mode 100644 index c4f4a31..0000000 --- a/docs/CSDK-guide.md +++ /dev/null @@ -1,197 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to you under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -# CSDK Guide - -CarbonData CSDK provides C++ interface to write and read carbon file. -CSDK use JNI to invoke java SDK in C++ code. - - -# CSDK Reader -This CSDK reader reads CarbonData file and carbonindex file at a given path. -External client can make use of this reader to read CarbonData files in C++ -code and without CarbonSession. - - -In the carbon jars package, there exist a carbondata-sdk.jar, -including SDK reader for CSDK. -## Quick example -``` -// 1. init JVM -JavaVM *jvm; -JNIEnv *initJVM() { - JNIEnv *env; - JavaVMInitArgs vm_args; - int parNum = 3; - int res; - JavaVMOption options[parNum]; - - options[0].optionString = "-Djava.compiler=NONE"; - options[1].optionString = "-Djava.class.path=../../sdk/target/carbondata-sdk.jar"; - options[2].optionString = "-verbose:jni"; - vm_args.version = JNI_VERSION_1_8; - vm_args.nOptions = parNum; - vm_args.options = options; - vm_args.ignoreUnrecognized = JNI_FALSE; - - res = JNI_CreateJavaVM(&jvm, (void **) &env, &vm_args); - if (res < 0) { - fprintf(stderr, "\nCan't create Java VM\n"); - exit(1); - } - - return env; -} - -// 2. create carbon reader and read data -// 2.1 read data from local disk -/** - * test read data from local disk, without projection - * - * @param env jni env - * @return - */ -bool readFromLocalWithoutProjection(JNIEnv *env) { - - CarbonReader carbonReaderClass; - carbonReaderClass.builder(env, "../resources/carbondata", "test"); - carbonReaderClass.build(); - - while (carbonReaderClass.hasNext()) { - jobjectArray row = carbonReaderClass.readNextRow(); - jsize length = env->GetArrayLength(row); - int j = 0; - for (j = 0; j < length; j++) { - jobject element = env->GetObjectArrayElement(row, j); - char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE); - printf("%s\t", str); - } - printf("\n"); - } - carbonReaderClass.close(); -} - -// 2.2 read data from S3 - -/** - * read data from S3 - * parameter is ak sk endpoint - * - * @param env jni env - * @param argv argument vector - * @return - */ -bool readFromS3(JNIEnv *env, char *argv[]) { - CarbonReader reader; - - char *args[3]; - // "your access key" - args[0] = argv[1]; - // "your secret key" - args[1] = argv[2]; - // "your endPoint" - args[2] = argv[3]; - - reader.builder(env, "s3a://sdk/WriterOutput", "test"); - reader.withHadoopConf(3, args); - reader.build(); - printf("\nRead data from S3:\n"); - while (reader.hasNext()) { - jobjectArray row = reader.readNextRow(); - jsize length = env->GetArrayLength(row); - - int j = 0; - for (j = 0; j < length; j++) { - jobject element = env->GetObjectArrayElement(row, j); - char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE); - printf("%s\t", str); - } - printf("\n"); - } - - reader.close(); -} - -// 3. destory JVM - (jvm)->DestroyJavaVM(); -``` -Find example code at main.cpp of CSDK module - -## API List -``` - /** - * create a CarbonReaderBuilder object for building carbonReader, - * CarbonReaderBuilder object can configure different parameter - * - * @param env JNIEnv - * @param path data store path - * @param tableName table name - * @return CarbonReaderBuilder object - */ - jobject builder(JNIEnv *env, char *path, char *tableName); - - /** - * Configure the projection column names of carbon reader - * - * @param argc argument counter - * @param argv argument vector - * @return CarbonReaderBuilder object - */ - jobject projection(int argc, char *argv[]); - - /** - * build carbon reader with argument vector - * it support multiple parameter - * like: key=value - * for example: fs.s3a.access.key=XXXX, XXXX is user's access key value - * - * @param argc argument counter - * @param argv argument vector - * @return CarbonReaderBuilder object - **/ - jobject withHadoopConf(int argc, char *argv[]); - - /** - * build carbonReader object for reading data - * it support read data from load disk - * - * @return carbonReader object - */ - jobject build(); - - /** - * Whether it has next row data - * - * @return boolean value, if it has next row, return true. if it hasn't next row, return false. - */ - jboolean hasNext(); - - /** - * read next row from data - * - * @return object array of one row - */ - jobjectArray readNextRow(); - - /** - * close the carbon reader - * - * @return boolean value - */ - jboolean close(); - -``` http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/docs/csdk-guide.md ---------------------------------------------------------------------- diff --git a/docs/csdk-guide.md b/docs/csdk-guide.md new file mode 100644 index 0000000..5598cb6 --- /dev/null +++ b/docs/csdk-guide.md @@ -0,0 +1,103 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to you under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +# CSDK Guide + +CarbonData CSDK provides C++ interface to write and read carbon file. +CSDK use JNI to invoke java SDK in C++ code. + + +# CSDK Reader +This CSDK reader reads CarbonData file and carbonindex file at a given path. +External client can make use of this reader to read CarbonData files in C++ +code and without CarbonSession. + + +In the carbon jars package, there exist a carbondata-sdk.jar, +including SDK reader for CSDK. +## Quick example + +Please find example code at [main.cpp](https://github.com/apache/carbondata/blob/master/store/CSDK/test/main.cpp) of CSDK module + +When users use C++ to read carbon files, users should init JVM first. Then users create +carbon reader and read data.There are some example code of read data from local disk +and read data from S3 at main.cpp of CSDK module. Finally, Finally, users need to +release the memory and destroy JVM. + +## API List +``` + /** + * create a CarbonReaderBuilder object for building carbonReader, + * CarbonReaderBuilder object can configure different parameter + * + * @param env JNIEnv + * @param path data store path + * @param tableName table name + * @return CarbonReaderBuilder object + */ + jobject builder(JNIEnv *env, char *path, char *tableName); + + /** + * Configure the projection column names of carbon reader + * + * @param argc argument counter + * @param argv argument vector + * @return CarbonReaderBuilder object + */ + jobject projection(int argc, char *argv[]); + + /** + * build carbon reader with argument vector + * it support multiple parameter + * like: key=value + * for example: fs.s3a.access.key=XXXX, XXXX is user's access key value + * + * @param argc argument counter + * @param argv argument vector + * @return CarbonReaderBuilder object + **/ + jobject withHadoopConf(int argc, char *argv[]); + + /** + * build carbonReader object for reading data + * it support read data from load disk + * + * @return carbonReader object + */ + jobject build(); + + /** + * Whether it has next row data + * + * @return boolean value, if it has next row, return true. if it hasn't next row, return false. + */ + jboolean hasNext(); + + /** + * read next carbonRow from data + * @return carbonRow object of one row + */ + jobject readNextRow(); + + /** + * close the carbon reader + * + * @return boolean value + */ + jboolean close(); + +``` http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/docs/documentation.md ---------------------------------------------------------------------- diff --git a/docs/documentation.md b/docs/documentation.md index 1b6726a..4a1d176 100644 --- a/docs/documentation.md +++ b/docs/documentation.md @@ -31,7 +31,7 @@ Apache CarbonData is a new big data file format for faster interactive query usi **CarbonData SQL Language Reference:** CarbonData extends the Spark SQL language and adds several [DDL](./ddl-of-carbondata.md) and [DML](./dml-of-carbondata.md) statements to support operations on it.Refer to the [Reference Manual](./language-manual.md) to understand the supported features and functions. -**Programming Guides:** You can read our guides about [APIs supported](./sdk-guide.md) to learn how to integrate CarbonData with your applications. +**Programming Guides:** You can read our guides about [Java APIs supported](./sdk-guide.md) or [C++ APIs supported](./csdk-guide.md) to learn how to integrate CarbonData with your applications. http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/CSDK/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/store/CSDK/CMakeLists.txt b/store/CSDK/CMakeLists.txt index c3c57a0..bfda148 100644 --- a/store/CSDK/CMakeLists.txt +++ b/store/CSDK/CMakeLists.txt @@ -1,17 +1,17 @@ -cmake_minimum_required (VERSION 2.8) -project (CJDK) +cmake_minimum_required(VERSION 2.8) +project(CJDK) set(CMAKE_BUILD_TYPE Debug) -SET (CMAKE_INSTALL_RPATH_USE_LINK_PATH true) +SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH true) -find_package (JNI REQUIRED) +find_package(JNI REQUIRED) -include_directories (${JNI_INCLUDE_DIRS}) +include_directories(${JNI_INCLUDE_DIRS}) -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") -set (SOURCE_FILES CarbonReader.cpp CarbonReader.h main.cpp) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +set(SOURCE_FILES src/CarbonReader.cpp src/CarbonReader.h test/main.cpp src/CarbonRow.h src/CarbonRow.cpp) -add_executable (CJDK ${SOURCE_FILES}) -get_filename_component (JAVA_JVM_LIBRARY_DIR ${JAVA_JVM_LIBRARY} DIRECTORY) -message (${JAVA_JVM_LIBRARY_DIR}) -target_link_libraries (CJDK ${JAVA_JVM_LIBRARY} ) +add_executable(CJDK ${SOURCE_FILES}) +get_filename_component(JAVA_JVM_LIBRARY_DIR ${JAVA_JVM_LIBRARY} DIRECTORY) +message(${JAVA_JVM_LIBRARY_DIR}) +target_link_libraries(CJDK ${JAVA_JVM_LIBRARY}) http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/CSDK/CarbonReader.cpp ---------------------------------------------------------------------- diff --git a/store/CSDK/CarbonReader.cpp b/store/CSDK/CarbonReader.cpp deleted file mode 100644 index 0400957..0000000 --- a/store/CSDK/CarbonReader.cpp +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "CarbonReader.h" -#include <jni.h> - -jobject CarbonReader::builder(JNIEnv *env, char *path, char *tableName) { - - jniEnv = env; - jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader"); - jmethodID carbonReaderBuilderID = env->GetStaticMethodID(carbonReaderClass, "builder", - "(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;"); - jstring jpath = env->NewStringUTF(path); - jstring jtableName = env->NewStringUTF(tableName); - jvalue args[2]; - args[0].l = jpath; - args[1].l = jtableName; - carbonReaderBuilderObject = env->CallStaticObjectMethodA(carbonReaderClass, carbonReaderBuilderID, args); - return carbonReaderBuilderObject; -} - -jobject CarbonReader::builder(JNIEnv *env, char *path) { - jniEnv = env; - jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader"); - jmethodID carbonReaderBuilderID = env->GetStaticMethodID(carbonReaderClass, "builder", - "(Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;"); - jstring jpath = env->NewStringUTF(path); - jvalue args[1]; - args[0].l = jpath; - carbonReaderBuilderObject = env->CallStaticObjectMethodA(carbonReaderClass, carbonReaderBuilderID, args); - return carbonReaderBuilderObject; -} - -jobject CarbonReader::projection(int argc, char *argv[]) { - jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject); - jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "projection", - "([Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;"); - jclass objectArrayClass = jniEnv->FindClass("Ljava/lang/String;"); - jobjectArray array = jniEnv->NewObjectArray(argc, objectArrayClass, NULL); - for (int i = 0; i < argc; ++i) { - jstring value = jniEnv->NewStringUTF(argv[i]); - jniEnv->SetObjectArrayElement(array, i, value); - } - - jvalue args[1]; - args[0].l = array; - carbonReaderBuilderObject = jniEnv->CallObjectMethodA(carbonReaderBuilderObject, buildID, args); - return carbonReaderBuilderObject; -} - -jobject CarbonReader::withHadoopConf(char *key, char *value) { - jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject); - jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "withHadoopConf", - "(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;"); - - jvalue args[2]; - args[0].l = jniEnv->NewStringUTF(key); - args[1].l = jniEnv->NewStringUTF(value); - carbonReaderBuilderObject = jniEnv->CallObjectMethodA(carbonReaderBuilderObject, buildID, args); - return carbonReaderBuilderObject; -} - -jobject CarbonReader::build() { - jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject); - jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "build", - "()Lorg/apache/carbondata/sdk/file/CarbonReader;"); - carbonReaderObject = jniEnv->CallObjectMethod(carbonReaderBuilderObject, buildID); - return carbonReaderObject; -} - -jboolean CarbonReader::hasNext() { - jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject); - jmethodID hasNextID = jniEnv->GetMethodID(carbonReader, "hasNext", "()Z"); - unsigned char hasNext = jniEnv->CallBooleanMethod(carbonReaderObject, hasNextID); - return hasNext; -} - -jobjectArray CarbonReader::readNextRow() { - jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject); - jmethodID readNextRow2ID = jniEnv->GetMethodID(carbonReader, "readNextStringRow", "()[Ljava/lang/Object;"); - jobjectArray row = (jobjectArray) jniEnv->CallObjectMethod(carbonReaderObject, readNextRow2ID); - return row; -} - -jboolean CarbonReader::close() { - jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject); - jmethodID closeID = jniEnv->GetMethodID(carbonReader, "close", "()V"); - jniEnv->CallBooleanMethod(carbonReaderObject, closeID); -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/CSDK/CarbonReader.h ---------------------------------------------------------------------- diff --git a/store/CSDK/CarbonReader.h b/store/CSDK/CarbonReader.h deleted file mode 100644 index 861f704..0000000 --- a/store/CSDK/CarbonReader.h +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <jni.h> - -class CarbonReader { -public: - /** - * jni env - */ - JNIEnv *jniEnv; - - /** - * carbonReaderBuilder object for building carbonReader - * it can configure some operation - */ - jobject carbonReaderBuilderObject; - - /** - * carbonReader object for reading data - */ - jobject carbonReaderObject; - - /** - * create a CarbonReaderBuilder object for building carbonReader, - * CarbonReaderBuilder object can configure different parameter - * - * @param env JNIEnv - * @param path data store path - * @param tableName table name - * @return CarbonReaderBuilder object - */ - jobject builder(JNIEnv *env, char *path, char *tableName); - - /** - * create a CarbonReaderBuilder object for building carbonReader, - * CarbonReaderBuilder object can configure different parameter - * - * @param env JNIEnv - * @param path data store path - * @return CarbonReaderBuilder object - * */ - jobject builder(JNIEnv *env, char *path); - - /** - * Configure the projection column names of carbon reader - * - * @param argc argument counter - * @param argv argument vector - * @return CarbonReaderBuilder object - */ - jobject projection(int argc, char *argv[]); - - /** - * configure parameter, including ak,sk and endpoint - * - * @param key key word - * @param value value - * @return CarbonReaderBuilder object - */ - jobject withHadoopConf(char *key, char *value); - - /** - * build carbonReader object for reading data - * it support read data from load disk - * - * @return carbonReader object - */ - jobject build(); - - /** - * Whether it has next row data - * - * @return boolean value, if it has next row, return true. if it hasn't next row, return false. - */ - jboolean hasNext(); - - /** - * read next row from data - * - * @return object array of one row - */ - jobjectArray readNextRow(); - - /** - * close the carbon reader - * - * @return boolean value - */ - jboolean close(); -}; http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/CSDK/main.cpp ---------------------------------------------------------------------- diff --git a/store/CSDK/main.cpp b/store/CSDK/main.cpp deleted file mode 100644 index a8d1a87..0000000 --- a/store/CSDK/main.cpp +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdio.h> -#include <jni.h> -#include <stdlib.h> -#include <iostream> -#include <unistd.h> -#include "CarbonReader.h" - -using namespace std; - -JavaVM *jvm; - -/** - * init jvm - * - * @return - */ -JNIEnv *initJVM() { - JNIEnv *env; - JavaVMInitArgs vm_args; - int parNum = 3; - int res; - JavaVMOption options[parNum]; - - options[0].optionString = "-Djava.compiler=NONE"; - options[1].optionString = "-Djava.class.path=../../sdk/target/carbondata-sdk.jar"; - options[2].optionString = "-verbose:jni"; - vm_args.version = JNI_VERSION_1_8; - vm_args.nOptions = parNum; - vm_args.options = options; - vm_args.ignoreUnrecognized = JNI_FALSE; - - res = JNI_CreateJavaVM(&jvm, (void **) &env, &vm_args); - if (res < 0) { - fprintf(stderr, "\nCan't create Java VM\n"); - exit(1); - } - - return env; -} - -/** - * test read data from local disk, without projection - * - * @param env jni env - * @return - */ -bool readFromLocalWithoutProjection(JNIEnv *env) { - - CarbonReader carbonReaderClass; - carbonReaderClass.builder(env, "../resources/carbondata"); - carbonReaderClass.build(); - - printf("\nRead data from local without projection:\n"); - - while (carbonReaderClass.hasNext()) { - jobjectArray row = carbonReaderClass.readNextRow(); - jsize length = env->GetArrayLength(row); - - int j = 0; - for (j = 0; j < length; j++) { - jobject element = env->GetObjectArrayElement(row, j); - char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE); - printf("%s\t", str); - } - printf("\n"); - } - - carbonReaderClass.close(); -} - -/** - * test read data from local disk - * - * @param env jni env - * @return - */ -bool readFromLocal(JNIEnv *env) { - - CarbonReader reader; - reader.builder(env, "../resources/carbondata", "test"); - - char *argv[11]; - argv[0] = "stringField"; - argv[1] = "shortField"; - argv[2] = "intField"; - argv[3] = "longField"; - argv[4] = "doubleField"; - argv[5] = "boolField"; - argv[6] = "dateField"; - argv[7] = "timeField"; - argv[8] = "decimalField"; - argv[9] = "varcharField"; - argv[10] = "arrayField"; - reader.projection(11, argv); - - reader.build(); - - printf("\nRead data from local:\n"); - - while (reader.hasNext()) { - jobjectArray row = reader.readNextRow(); - jsize length = env->GetArrayLength(row); - - int j = 0; - for (j = 0; j < length; j++) { - jobject element = env->GetObjectArrayElement(row, j); - char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE); - printf("%s\t", str); - } - printf("\n"); - } - - reader.close(); -} - -/** - * read data from S3 - * parameter is ak sk endpoint - * - * @param env jni env - * @param argv argument vector - * @return - */ -bool readFromS3(JNIEnv *env, char *argv[]) { - CarbonReader reader; - - char *args[3]; - // "your access key" - args[0] = argv[1]; - // "your secret key" - args[1] = argv[2]; - // "your endPoint" - args[2] = argv[3]; - - reader.builder(env, "s3a://sdk/WriterOutput", "test"); - reader.withHadoopConf("fs.s3a.access.key", argv[1]); - reader.withHadoopConf("fs.s3a.secret.key", argv[2]); - reader.withHadoopConf("fs.s3a.endpoint", argv[3]); - reader.build(); - printf("\nRead data from S3:\n"); - while (reader.hasNext()) { - jobjectArray row = reader.readNextRow(); - jsize length = env->GetArrayLength(row); - - int j = 0; - for (j = 0; j < length; j++) { - jobject element = env->GetObjectArrayElement(row, j); - char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE); - printf("%s\t", str); - } - printf("\n"); - } - - reader.close(); -} - -/** - * This a example for C++ interface to read carbon file - * If you want to test read data fromS3, please input the parameter: ak sk endpoint - * - * @param argc argument counter - * @param argv argument vector - * @return - */ -int main(int argc, char *argv[]) { - // init jvm - JNIEnv *env; - env = initJVM(); - - if (argc > 3) { - readFromS3(env, argv); - } else { - readFromLocalWithoutProjection(env); - readFromLocal(env); - } - cout << "destory jvm\n\n"; - (jvm)->DestroyJavaVM(); - - cout << "\nfinish destory jvm"; - fprintf(stdout, "Java VM destory.\n"); - return 0; -} - http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/CSDK/src/CarbonReader.cpp ---------------------------------------------------------------------- diff --git a/store/CSDK/src/CarbonReader.cpp b/store/CSDK/src/CarbonReader.cpp new file mode 100644 index 0000000..456e00e --- /dev/null +++ b/store/CSDK/src/CarbonReader.cpp @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "CarbonReader.h" +#include <jni.h> +#include <mach/mach_types.h> +#include <stdexcept> + +void CarbonReader::builder(JNIEnv *env, char *path, char *tableName) { + if (env == NULL) { + throw std::runtime_error("JNIEnv parameter can't be NULL."); + } + if (path == NULL) { + throw std::runtime_error("path parameter can't be NULL."); + } + if (tableName == NULL) { + throw std::runtime_error("tableName parameter can't be NULL."); + } + jniEnv = env; + jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader"); + if (carbonReaderClass == NULL) { + throw std::runtime_error("Can't find the class in java: org/apache/carbondata/sdk/file/CarbonReader"); + } + jmethodID carbonReaderBuilderID = env->GetStaticMethodID(carbonReaderClass, "builder", + "(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;"); + if (carbonReaderBuilderID == NULL) { + throw std::runtime_error("Can't find the method in java: builder"); + } + jstring jPath = env->NewStringUTF(path); + jstring jTableName = env->NewStringUTF(tableName); + jvalue args[2]; + args[0].l = jPath; + args[1].l = jTableName; + carbonReaderBuilderObject = env->CallStaticObjectMethodA(carbonReaderClass, carbonReaderBuilderID, args); +} + +void CarbonReader::builder(JNIEnv *env, char *path) { + if (env == NULL) { + throw std::runtime_error("JNIEnv parameter can't be NULL."); + } + if (path == NULL) { + throw std::runtime_error("path parameter can't be NULL."); + } + jniEnv = env; + jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader"); + if (carbonReaderClass == NULL) { + throw std::runtime_error("Can't find the class in java: org/apache/carbondata/sdk/file/CarbonReader"); + } + jmethodID carbonReaderBuilderID = env->GetStaticMethodID(carbonReaderClass, "builder", + "(Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;"); + if (carbonReaderBuilderID == NULL) { + throw std::runtime_error("Can't find the method in java: builder"); + } + jstring jPath = env->NewStringUTF(path); + jvalue args[1]; + args[0].l = jPath; + carbonReaderBuilderObject = env->CallStaticObjectMethodA(carbonReaderClass, carbonReaderBuilderID, args); +} + +void CarbonReader::projection(int argc, char *argv[]) { + if (argc < 0) { + throw std::runtime_error("argc parameter can't be negative."); + } + if (argv == NULL) { + throw std::runtime_error("argv parameter can't be NULL."); + } + jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject); + jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "projection", + "([Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;"); + if (buildID == NULL) { + throw std::runtime_error("Can't find the method in java: projection"); + } + jclass objectArrayClass = jniEnv->FindClass("Ljava/lang/String;"); + if (objectArrayClass == NULL) { + throw std::runtime_error("Can't find the class in java: java/lang/String"); + } + jobjectArray array = jniEnv->NewObjectArray(argc, objectArrayClass, NULL); + for (int i = 0; i < argc; ++i) { + jstring value = jniEnv->NewStringUTF(argv[i]); + jniEnv->SetObjectArrayElement(array, i, value); + } + + jvalue args[1]; + args[0].l = array; + carbonReaderBuilderObject = jniEnv->CallObjectMethodA(carbonReaderBuilderObject, buildID, args); +} + +void CarbonReader::withHadoopConf(char *key, char *value) { + if (key == NULL) { + throw std::runtime_error("key parameter can't be NULL."); + } + if (value == NULL) { + throw std::runtime_error("value parameter can't be NULL."); + } + jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject); + jmethodID id = jniEnv->GetMethodID(carbonReaderBuilderClass, "withHadoopConf", + "(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;"); + if (id == NULL) { + throw std::runtime_error("Can't find the method in java: withHadoopConf"); + } + jvalue args[2]; + args[0].l = jniEnv->NewStringUTF(key); + args[1].l = jniEnv->NewStringUTF(value); + carbonReaderBuilderObject = jniEnv->CallObjectMethodA(carbonReaderBuilderObject, id, args); +} + +jobject CarbonReader::build() { + jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject); + jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "build", + "()Lorg/apache/carbondata/sdk/file/CarbonReader;"); + if (buildID == NULL) { + throw std::runtime_error("Can't find the method in java: build"); + } + carbonReaderObject = jniEnv->CallObjectMethod(carbonReaderBuilderObject, buildID); + if (jniEnv->ExceptionCheck()) { + throw jniEnv->ExceptionOccurred(); + } + return carbonReaderObject; +} + +jboolean CarbonReader::hasNext() { + if (hasNextID == NULL) { + jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject); + hasNextID = jniEnv->GetMethodID(carbonReader, "hasNext", "()Z"); + if (hasNextID == NULL) { + throw std::runtime_error("Can't find the method in java: hasNext"); + } + } + jboolean result = jniEnv->CallBooleanMethod(carbonReaderObject, hasNextID); + if (jniEnv->ExceptionCheck()) { + throw jniEnv->ExceptionOccurred(); + } + return result; +} + +jobject CarbonReader::readNextRow() { + if (readNextRowID == NULL) { + jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject); + readNextRowID = jniEnv->GetMethodID(carbonReader, "readNextRow", + "()Ljava/lang/Object;"); + if (readNextRowID == NULL) { + throw std::runtime_error("Can't find the method in java: readNextRow"); + } + } + jobject result = jniEnv->CallObjectMethod(carbonReaderObject, readNextRowID); + if (jniEnv->ExceptionCheck()) { + throw jniEnv->ExceptionOccurred(); + } + return result; +} + +void CarbonReader::close() { + jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject); + jmethodID closeID = jniEnv->GetMethodID(carbonReader, "close", "()V"); + if (closeID == NULL) { + throw std::runtime_error("Can't find the method in java: close"); + } + jniEnv->CallBooleanMethod(carbonReaderObject, closeID); + if (jniEnv->ExceptionCheck()) { + throw jniEnv->ExceptionOccurred(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/CSDK/src/CarbonReader.h ---------------------------------------------------------------------- diff --git a/store/CSDK/src/CarbonReader.h b/store/CSDK/src/CarbonReader.h new file mode 100644 index 0000000..b4841bf --- /dev/null +++ b/store/CSDK/src/CarbonReader.h @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <jni.h> + +class CarbonReader { +private: + /** + * hasNext jmethodID + */ + jmethodID hasNextID = NULL; + + /** + * readNextRow jmethodID + */ + jmethodID readNextRowID = NULL; + + /** + * carbonReaderBuilder object for building carbonReader + * it can configure some operation + */ + jobject carbonReaderBuilderObject; + + /** + * carbonReader object for reading data + */ + jobject carbonReaderObject; + +public: + + /** + * jni env + */ + JNIEnv *jniEnv; + + /** + * create a CarbonReaderBuilder object for building carbonReader, + * CarbonReaderBuilder object can configure different parameter + * + * @param env JNIEnv + * @param path data store path + * @param tableName table name + */ + void builder(JNIEnv *env, char *path, char *tableName); + + /** + * create a CarbonReaderBuilder object for building carbonReader, + * CarbonReaderBuilder object can configure different parameter + * + * @param env JNIEnv + * @param path data store path + * */ + void builder(JNIEnv *env, char *path); + + /** + * Configure the projection column names of carbon reader + * + * @param argc argument counter + * @param argv argument vector + */ + void projection(int argc, char *argv[]); + + /** + * configure parameter, including ak,sk and endpoint + * + * @param key key word + * @param value value + */ + void withHadoopConf(char *key, char *value); + + /** + * build carbonReader object for reading data + * it support read data from load disk + * + * @return carbonReader object + */ + jobject build(); + + /** + * Whether it has next row data + * + * @return boolean value, if it has next row, return true. if it hasn't next row, return false. + */ + jboolean hasNext(); + + /** + * read next carbon Row from data + * @return carbon Row object of one row + */ + jobject readNextRow(); + + /** + * close the carbon reader + * + * @return boolean value + */ + void close(); +}; http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/CSDK/src/CarbonRow.cpp ---------------------------------------------------------------------- diff --git a/store/CSDK/src/CarbonRow.cpp b/store/CSDK/src/CarbonRow.cpp new file mode 100644 index 0000000..1c18ae6 --- /dev/null +++ b/store/CSDK/src/CarbonRow.cpp @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <jni.h> +#include <cstring> +#include <stdexcept> +#include "CarbonRow.h" + +CarbonRow::CarbonRow(JNIEnv *env) { + if (env == NULL) { + throw std::runtime_error("JNIEnv parameter can't be NULL."); + } + this->rowUtilClass = env->FindClass("org/apache/carbondata/sdk/file/RowUtil"); + if (rowUtilClass == NULL) { + throw std::runtime_error("Can't find the class in java: org/apache/carbondata/sdk/file/RowUtil"); + } + this->jniEnv = env; + getShortId = jniEnv->GetStaticMethodID(rowUtilClass, "getShort", + "([Ljava/lang/Object;I)S"); + if (getShortId == NULL) { + throw std::runtime_error("Can't find the method in java: getShort"); + } + + getIntId = jniEnv->GetStaticMethodID(rowUtilClass, "getInt", + "([Ljava/lang/Object;I)I"); + if (getIntId == NULL) { + throw std::runtime_error("Can't find the method in java: getInt"); + } + getLongId = jniEnv->GetStaticMethodID(rowUtilClass, "getLong", + "([Ljava/lang/Object;I)J"); + if (getLongId == NULL) { + throw std::runtime_error("Can't find the method in java: getLong"); + } + + getDoubleId = jniEnv->GetStaticMethodID(rowUtilClass, "getDouble", + "([Ljava/lang/Object;I)D"); + if (getDoubleId == NULL) { + throw std::runtime_error("Can't find the method in java: getDouble"); + } + + getFloatId = jniEnv->GetStaticMethodID(rowUtilClass, "getFloat", + "([Ljava/lang/Object;I)F"); + if (getFloatId == NULL) { + throw std::runtime_error("Can't find the method in java: getFloat"); + } + + getBooleanId = jniEnv->GetStaticMethodID(rowUtilClass, "getBoolean", + "([Ljava/lang/Object;I)Z"); + if (getBooleanId == NULL) { + throw std::runtime_error("Can't find the method in java: getBoolean"); + } + + getStringId = jniEnv->GetStaticMethodID(rowUtilClass, "getString", + "([Ljava/lang/Object;I)Ljava/lang/String;"); + if (getStringId == NULL) { + throw std::runtime_error("Can't find the method in java: getString"); + } + + getDecimalId = jniEnv->GetStaticMethodID(rowUtilClass, "getDecimal", + "([Ljava/lang/Object;I)Ljava/lang/String;"); + if (getDecimalId == NULL) { + throw std::runtime_error("Can't find the method in java: getDecimal"); + } + + getVarcharId = jniEnv->GetStaticMethodID(rowUtilClass, "getVarchar", + "([Ljava/lang/Object;I)Ljava/lang/String;"); + if (getVarcharId == NULL) { + throw std::runtime_error("Can't find the method in java: getVarchar"); + } + + getArrayId = jniEnv->GetStaticMethodID(rowUtilClass, "getArray", + "([Ljava/lang/Object;I)[Ljava/lang/Object;"); + if (getArrayId == NULL) { + throw std::runtime_error("Can't find the method in java: getArray"); + } +} + +void CarbonRow::setCarbonRow(jobject data) { + if (data == NULL) { + throw std::runtime_error("data parameter can't be NULL."); + } + this->carbonRow = data; +} + +void CarbonRow::checkOrdinal(int ordinal) { + if (ordinal < 0) { + throw std::runtime_error("ordinal parameter can't be negative."); + } +} + +short CarbonRow::getShort(int ordinal) { + checkOrdinal(ordinal); + jvalue args[2]; + args[0].l = carbonRow; + args[1].i = ordinal; + return jniEnv->CallStaticShortMethodA(rowUtilClass, getShortId, args); +} + +int CarbonRow::getInt(int ordinal) { + checkOrdinal(ordinal); + jvalue args[2]; + args[0].l = carbonRow; + args[1].i = ordinal; + return jniEnv->CallStaticIntMethodA(rowUtilClass, getIntId, args); +} + +long CarbonRow::getLong(int ordinal) { + checkOrdinal(ordinal); + jvalue args[2]; + args[0].l = carbonRow; + args[1].i = ordinal; + return jniEnv->CallStaticLongMethodA(rowUtilClass, getLongId, args); +} + +double CarbonRow::getDouble(int ordinal) { + checkOrdinal(ordinal); + jvalue args[2]; + args[0].l = carbonRow; + args[1].i = ordinal; + return jniEnv->CallStaticDoubleMethodA(rowUtilClass, getDoubleId, args); +} + + +float CarbonRow::getFloat(int ordinal) { + checkOrdinal(ordinal); + jvalue args[2]; + args[0].l = carbonRow; + args[1].i = ordinal; + return jniEnv->CallStaticFloatMethodA(rowUtilClass, getFloatId, args); +} + +jboolean CarbonRow::getBoolean(int ordinal) { + checkOrdinal(ordinal); + jvalue args[2]; + args[0].l = carbonRow; + args[1].i = ordinal; + return jniEnv->CallStaticBooleanMethodA(rowUtilClass, getBooleanId, args); +} + +char *CarbonRow::getString(int ordinal) { + checkOrdinal(ordinal); + jvalue args[2]; + args[0].l = carbonRow; + args[1].i = ordinal; + jobject data = jniEnv->CallStaticObjectMethodA(rowUtilClass, getStringId, args); + char *str = (char *) jniEnv->GetStringUTFChars((jstring) data, JNI_FALSE); + jniEnv->DeleteLocalRef(data); + return str; +} + +char *CarbonRow::getDecimal(int ordinal) { + checkOrdinal(ordinal); + jvalue args[2]; + args[0].l = carbonRow; + args[1].i = ordinal; + jobject data = jniEnv->CallStaticObjectMethodA(rowUtilClass, getDecimalId, args); + char *str = (char *) jniEnv->GetStringUTFChars((jstring) data, JNI_FALSE); + jniEnv->DeleteLocalRef(data); + return str; +} + +char *CarbonRow::getVarchar(int ordinal) { + checkOrdinal(ordinal); + jvalue args[2]; + args[0].l = carbonRow; + args[1].i = ordinal; + jobject data = jniEnv->CallStaticObjectMethodA(rowUtilClass, getVarcharId, args); + char *str = (char *) jniEnv->GetStringUTFChars((jstring) data, JNI_FALSE); + jniEnv->DeleteLocalRef(data); + return str; +} + +jobjectArray CarbonRow::getArray(int ordinal) { + checkOrdinal(ordinal); + jvalue args[2]; + args[0].l = carbonRow; + args[1].i = ordinal; + return (jobjectArray) jniEnv->CallStaticObjectMethodA(rowUtilClass, getArrayId, args); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/CSDK/src/CarbonRow.h ---------------------------------------------------------------------- diff --git a/store/CSDK/src/CarbonRow.h b/store/CSDK/src/CarbonRow.h new file mode 100644 index 0000000..007f8a9 --- /dev/null +++ b/store/CSDK/src/CarbonRow.h @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <jni.h> + +class CarbonRow { +private: + jmethodID getShortId; + jmethodID getIntId; + jmethodID getLongId; + jmethodID getDoubleId; + jmethodID getFloatId; + jmethodID getBooleanId; + jmethodID getStringId; + jmethodID getDecimalId; + jmethodID getVarcharId; + jmethodID getArrayId; + + /** + * RowUtil Class for read data from Carbon Row + */ + jclass rowUtilClass; + + /** + * carbon row data + */ + jobject carbonRow; + + /** + * check ordinal, ordinal can't be negative + * + * @param ordinal int value, the data index of carbon Row + */ + void checkOrdinal(int ordinal); + +public: + + /** + * jni env + */ + JNIEnv *jniEnv; + + /** + * Constructor and express the carbon row result + * + * @param env JNI env + * @param jo carbon Row object + */ + CarbonRow(JNIEnv *env); + + /** + * set carbon row data + * + * @param data + */ + void setCarbonRow(jobject data); + + /** + * get short data type data by ordinal + * + * @param ordinal the data index of carbon Row + * @return short data type data + */ + short getShort(int ordinal); + + /** + * get int data type data by ordinal + * + * @param ordinal the data index of carbon Row + * @return int data type data + */ + int getInt(int ordinal); + + /** + * get long data type data by ordinal + * + * @param ordinal the data index of carbon Row + * @return long data type data + */ + long getLong(int ordinal); + + /** + * get double data type data by ordinal + * + * @param ordinal the data index of carbon Row + * @return double data type data + */ + double getDouble(int ordinal); + + /** + * get float data type data by ordinal + * + * @param ordinal the data index of carbon Row + * @return float data type data + */ + float getFloat(int ordinal); + + /** + * get boolean data type data by ordinal + * + * @param ordinal the data index of carbon Row + * @return jboolean data type data + */ + jboolean getBoolean(int ordinal); + + /** + * get decimal data type data by ordinal + * JNI don't support Decimal, so carbon convert decimal to string + * + * @param ordinal the data index of carbon Row + * @return string data type data + */ + char *getDecimal(int ordinal); + + /** + * get string data type data by ordinal + * + * @param ordinal the data index of carbon Row + * @return string data type data + */ + char *getString(int ordinal); + + /** + * get varchar data type data by ordinal + * JNI don't support varchar, so carbon convert decimal to string + * + * @param ordinal the data index of carbon Row + * @return string data type data + */ + char *getVarchar(int ordinal); + + /** + * get array<T> data type data by ordinal + * + * @param ordinal the data index of carbon Row + * @return jobjectArray data type data + */ + jobjectArray getArray(int ordinal); +}; http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/CSDK/test/main.cpp ---------------------------------------------------------------------- diff --git a/store/CSDK/test/main.cpp b/store/CSDK/test/main.cpp new file mode 100644 index 0000000..843155f --- /dev/null +++ b/store/CSDK/test/main.cpp @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdio.h> +#include <jni.h> +#include <stdlib.h> +#include <iostream> +#include <unistd.h> +#include "../src/CarbonReader.h" +#include "../src/CarbonRow.h" + +using namespace std; + +JavaVM *jvm; + +/** + * init jvm + * + * @return + */ +JNIEnv *initJVM() { + JNIEnv *env; + JavaVMInitArgs vm_args; + int parNum = 3; + int res; + JavaVMOption options[parNum]; + + options[0].optionString = "-Djava.compiler=NONE"; + options[1].optionString = "-Djava.class.path=../../sdk/target/carbondata-sdk.jar"; + options[2].optionString = "-verbose:jni"; + vm_args.version = JNI_VERSION_1_8; + vm_args.nOptions = parNum; + vm_args.options = options; + vm_args.ignoreUnrecognized = JNI_FALSE; + + res = JNI_CreateJavaVM(&jvm, (void **) &env, &vm_args); + if (res < 0) { + fprintf(stderr, "\nCan't create Java VM\n"); + exit(1); + } + + return env; +} + +/** + * print array result + * + * @param env JNIEnv + * @param arr array + */ +void printArray(JNIEnv *env, jobjectArray arr) { + jsize length = env->GetArrayLength(arr); + int j = 0; + for (j = 0; j < length; j++) { + jobject element = env->GetObjectArrayElement(arr, j); + char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE); + printf("%s\t", str); + } + env->DeleteLocalRef(arr); +} + +/** + * print boolean result + * + * @param env JNIEnv + * @param bool1 boolean value + */ +void printBoolean(jboolean bool1) { + if (bool1) { + printf("true\t"); + } else { + printf("false\t"); + } +} + +/** + * print result of reading data + * + * @param env JNIEnv + * @param reader CarbonReader object + */ +void printResult(JNIEnv *env, CarbonReader reader) { + CarbonRow carbonRow(env); + while (reader.hasNext()) { + jobject row = reader.readNextRow(); + carbonRow.setCarbonRow(row); + printf("%s\t", carbonRow.getString(0)); + printf("%d\t", carbonRow.getInt(1)); + printf("%ld\t", carbonRow.getLong(2)); + printf("%s\t", carbonRow.getVarchar(3)); + printArray(env, carbonRow.getArray(4)); + printf("%d\t", carbonRow.getShort(5)); + printf("%d\t", carbonRow.getInt(6)); + printf("%ld\t", carbonRow.getLong(7)); + printf("%lf\t", carbonRow.getDouble(8)); + printBoolean(carbonRow.getBoolean(9)); + printf("%s\t", carbonRow.getDecimal(10)); + printf("%f\t", carbonRow.getFloat(11)); + printf("\n"); + env->DeleteLocalRef(row); + } + reader.close(); +} + +/** + * test read data from local disk, without projection + * + * @param env jni env + * @return + */ +bool readFromLocalWithoutProjection(JNIEnv *env) { + printf("\nRead data from local without projection:\n"); + + CarbonReader carbonReaderClass; + try { + carbonReaderClass.builder(env, "../../../../resources/carbondata"); + } catch (runtime_error e) { + printf("\nget exception fro builder and throw\n"); + throw e; + } + try { + carbonReaderClass.build(); + } catch (jthrowable e) { + env->ExceptionDescribe(); + } + printResult(env, carbonReaderClass); +} + +/** + * test read data from local disk + * + * @param env jni env + * @return + */ +bool readFromLocal(JNIEnv *env) { + printf("\nRead data from local:\n"); + + CarbonReader reader; + reader.builder(env, "../../../../resources/carbondata", "test"); + + char *argv[12]; + argv[0] = "stringField"; + argv[1] = "shortField"; + argv[2] = "intField"; + argv[3] = "longField"; + argv[4] = "doubleField"; + argv[5] = "boolField"; + argv[6] = "dateField"; + argv[7] = "timeField"; + argv[8] = "decimalField"; + argv[9] = "varcharField"; + argv[10] = "arrayField"; + argv[11] = "floatField"; + reader.projection(12, argv); + + try { + reader.build(); + } catch (jthrowable e) { + env->ExceptionDescribe(); + } + + CarbonRow carbonRow(env); + while (reader.hasNext()) { + jobject row = reader.readNextRow(); + carbonRow.setCarbonRow(row); + + printf("%s\t", carbonRow.getString(0)); + printf("%d\t", carbonRow.getShort(1)); + printf("%d\t", carbonRow.getInt(2)); + printf("%ld\t", carbonRow.getLong(3)); + printf("%lf\t", carbonRow.getDouble(4)); + printBoolean(carbonRow.getBoolean(5)); + printf("%d\t", carbonRow.getInt(6)); + printf("%ld\t", carbonRow.getLong(7)); + printf("%s\t", carbonRow.getDecimal(8)); + printf("%s\t", carbonRow.getVarchar(9)); + printArray(env, carbonRow.getArray(10)); + printf("%f\t", carbonRow.getFloat(11)); + printf("\n"); + env->DeleteLocalRef(row); + } + + reader.close(); +} + + +bool tryCatchException(JNIEnv *env) { + printf("\ntry catch exception and print:\n"); + + CarbonReader carbonReaderClass; + carbonReaderClass.builder(env, "./carbondata"); + try { + carbonReaderClass.build(); + } catch (jthrowable e) { + env->ExceptionDescribe(); + } + printf("\nfinished handle exception\n"); +} +/** + * read data from S3 + * parameter is ak sk endpoint + * + * @param env jni env + * @param argv argument vector + * @return + */ +bool readFromS3(JNIEnv *env, char *argv[]) { + printf("\nRead data from S3:\n"); + CarbonReader reader; + + char *args[3]; + // "your access key" + args[0] = argv[1]; + // "your secret key" + args[1] = argv[2]; + // "your endPoint" + args[2] = argv[3]; + + reader.builder(env, "s3a://sdk/WriterOutput/carbondata/", "test"); + reader.withHadoopConf("fs.s3a.access.key", argv[1]); + reader.withHadoopConf("fs.s3a.secret.key", argv[2]); + reader.withHadoopConf("fs.s3a.endpoint", argv[3]); + reader.build(); + printResult(env, reader); +} + +/** + * This a example for C++ interface to read carbon file + * If you want to test read data fromS3, please input the parameter: ak sk endpoint + * + * @param argc argument counter + * @param argv argument vector + * @return + */ +int main(int argc, char *argv[]) { + // init jvm + JNIEnv *env; + env = initJVM(); + + if (argc > 3) { + readFromS3(env, argv); + } else { + tryCatchException(env); + readFromLocalWithoutProjection(env); + readFromLocal(env); + } + (jvm)->DestroyJavaVM(); + + cout << "\nfinish destroy jvm"; + return 0; +} + http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java index 932cf85..a381429 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java @@ -23,7 +23,6 @@ import java.util.UUID; import org.apache.carbondata.common.annotations.InterfaceAudience; import org.apache.carbondata.common.annotations.InterfaceStability; -import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.util.CarbonTaskInfo; import org.apache.carbondata.core.util.CarbonUtil; import org.apache.carbondata.core.util.ThreadLocalTaskInfo; @@ -92,35 +91,6 @@ public class CarbonReader<T> { } /** - * Read and return next string row object - * limitation: only single dimension Array is supported - * TODO: support didfferent data type - */ - public Object[] readNextStringRow() throws IOException, InterruptedException { - validateReader(); - T t = currentReader.getCurrentValue(); - Object[] objects = (Object[]) t; - String[] strings = new String[objects.length]; - for (int i = 0; i < objects.length; i++) { - if (objects[i] instanceof Object[]) { - Object[] arrayString = (Object[]) objects[i]; - StringBuffer stringBuffer = new StringBuffer(); - stringBuffer.append(String.valueOf(arrayString[0])); - if (arrayString.length > 1) { - for (int j = 1; j < arrayString.length; j++) { - stringBuffer.append(CarbonCommonConstants.ARRAY_SEPARATOR) - .append(String.valueOf(arrayString[j])); - } - } - strings[i] = stringBuffer.toString(); - } else { - strings[i] = String.valueOf(objects[i]); - } - } - return strings; - } - - /** * Return a new {@link CarbonReaderBuilder} instance * * @param tablePath table store path http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java index 924835f..065ff80 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java @@ -240,6 +240,15 @@ public class Field { } } + @Override + public String toString() { + return "Field{" + + "name='" + name + '\'' + + ", type=" + type + + ", schemaOrdinal=" + schemaOrdinal + + '}'; + } + /** * prepare sub fields for complex types * http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/sdk/src/main/java/org/apache/carbondata/sdk/file/RowUtil.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/RowUtil.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/RowUtil.java new file mode 100644 index 0000000..82b3904 --- /dev/null +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/RowUtil.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.sdk.file; + +import java.io.Serializable; +import java.math.BigDecimal; + +/** + * This row class is used to transfer the row data from one step to other step + */ +public class RowUtil implements Serializable { + + public static String getString(Object[] data, int ordinal) { + return (String) data[ordinal]; + } + + /** + * get short type data by ordinal + * + * @param data carbon row data + * @param ordinal the data index of Row + * @return short data type data + */ + public static short getShort(Object[] data, int ordinal) { + return (short) data[ordinal]; + } + + /** + * get int data type data by ordinal + * + * @param data carbon row data + * @param ordinal the data index of Row + * @return int data type data + */ + public static int getInt(Object[] data, int ordinal) { + return (Integer) data[ordinal]; + } + + /** + * get long data type data by ordinal + * + * @param data carbon row data + * @param ordinal the data index of Row + * @return long data type data + */ + public static long getLong(Object[] data, int ordinal) { + return (long) data[ordinal]; + } + + /** + * get array data type data by ordinal + * + * @param data carbon row data + * @param ordinal the data index of Row + * @return array data type data + */ + public static Object[] getArray(Object[] data, int ordinal) { + return (Object[]) data[ordinal]; + } + + /** + * get double data type data by ordinal + * + * @param data carbon row data + * @param ordinal the data index of Row + * @return double data type data + */ + public static double getDouble(Object[] data, int ordinal) { + return (double) data[ordinal]; + } + + /** + * get boolean data type data by ordinal + * + * @param data carbon row data + * @param ordinal the data index of Row + * @return boolean data type data + */ + public static boolean getBoolean(Object[] data, int ordinal) { + return (boolean) data[ordinal]; + } + + /** + * get byte data type data by ordinal + * + * @param data carbon row data + * @param ordinal the data index of Row + * @return byte data type data + */ + public static Byte getByte(Object[] data, int ordinal) { + return (Byte) data[ordinal]; + } + + /** + * get float data type data by ordinal + * + * @param data carbon row data + * @param ordinal the data index of Row + * @return float data type data + */ + public static float getFloat(Object[] data, int ordinal) { + return (float) data[ordinal]; + } + + /** + * get varchar data type data by ordinal + * This is for CSDK + * JNI don't support varchar, so carbon convert varchar to string + * + * @param data carbon row data + * @param ordinal the data index of Row + * @return string data type data + */ + public static String getVarchar(Object[] data, int ordinal) { + return (String) data[ordinal]; + } + + /** + * get decimal data type data by ordinal + * This is for CSDK + * JNI don't support Decimal, so carbon convert decimal to string + * + * @param data carbon row data + * @param ordinal the data index of Row + * @return string data type data + */ + public static String getDecimal(Object[] data, int ordinal) { + return ((BigDecimal) data[ordinal]).toString(); + } + +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java index f740ae2..09d664f 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java @@ -61,7 +61,8 @@ public class AvroCarbonWriterTest { } @After - public void verifyDMFile() { + public void verifyDMFile() throws IOException { + FileUtils.deleteDirectory(new File(path)); assert (!TestUtil.verifyMdtFile()); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java index c8b2c96..b8eb224 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java @@ -1529,4 +1529,212 @@ public class CarbonReaderTest extends TestCase { e.printStackTrace(); } } + + @Test + public void testReadNextRowWithRowUtil() { + String path = "./carbondata"; + try { + FileUtils.deleteDirectory(new File(path)); + + Field[] fields = new Field[12]; + fields[0] = new Field("stringField", DataTypes.STRING); + fields[1] = new Field("shortField", DataTypes.SHORT); + fields[2] = new Field("intField", DataTypes.INT); + fields[3] = new Field("longField", DataTypes.LONG); + fields[4] = new Field("doubleField", DataTypes.DOUBLE); + fields[5] = new Field("boolField", DataTypes.BOOLEAN); + fields[6] = new Field("dateField", DataTypes.DATE); + fields[7] = new Field("timeField", DataTypes.TIMESTAMP); + fields[8] = new Field("decimalField", DataTypes.createDecimalType(8, 2)); + fields[9] = new Field("varcharField", DataTypes.VARCHAR); + fields[10] = new Field("arrayField", DataTypes.createArrayType(DataTypes.STRING)); + fields[11] = new Field("floatField", DataTypes.FLOAT); + Map<String, String> map = new HashMap<>(); + map.put("complex_delimiter_level_1", "#"); + CarbonWriter writer = CarbonWriter.builder() + .outputPath(path) + .withLoadOptions(map) + .withCsvInput(new Schema(fields)) + .writtenBy("CarbonReaderTest") + .build(); + + for (int i = 0; i < 10; i++) { + String[] row2 = new String[]{ + "robot" + (i % 10), + String.valueOf(i % 10000), + String.valueOf(i), + String.valueOf(Long.MAX_VALUE - i), + String.valueOf((double) i / 2), + String.valueOf(true), + "2019-03-02", + "2019-02-12 03:03:34", + "12.345", + "varchar", + "Hello#World#From#Carbon", + "1.23" + }; + writer.write(row2); + } + writer.close(); + + File[] dataFiles = new File(path).listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + if (name == null) { + return false; + } + return name.endsWith("carbonindex"); + } + }); + if (dataFiles == null || dataFiles.length < 1) { + throw new RuntimeException("Carbon index file not exists."); + } + Schema schema = CarbonSchemaReader + .readSchemaInIndexFile(dataFiles[0].getAbsolutePath()) + .asOriginOrder(); + // Transform the schema + int count = 0; + for (int i = 0; i < schema.getFields().length; i++) { + if (!((schema.getFields())[i].getFieldName().contains("."))) { + count++; + } + } + String[] strings = new String[count]; + int index = 0; + for (int i = 0; i < schema.getFields().length; i++) { + if (!((schema.getFields())[i].getFieldName().contains("."))) { + strings[index] = (schema.getFields())[i].getFieldName(); + index++; + } + } + // Read data + CarbonReader reader = CarbonReader + .builder(path, "_temp") + .projection(strings) + .build(); + + int i = 0; + while (reader.hasNext()) { + Object[] data = (Object[]) reader.readNextRow(); + + assert (RowUtil.getString(data, 0).equals("robot" + i)); + assertEquals(RowUtil.getShort(data, 1), i); + assertEquals(RowUtil.getInt(data, 2), i); + assertEquals(RowUtil.getLong(data, 3), Long.MAX_VALUE - i); + assertEquals(RowUtil.getDouble(data, 4), ((double) i) / 2); + assert (RowUtil.getBoolean(data, 5)); + assertEquals(RowUtil.getInt(data, 6), 17957); + assert (RowUtil.getDecimal(data, 8).equals("12.35")); + assert (RowUtil.getVarchar(data, 9).equals("varchar")); + + Object[] arr = RowUtil.getArray(data, 10); + assert (arr[0].equals("Hello")); + assert (arr[1].equals("World")); + assert (arr[2].equals("From")); + assert (arr[3].equals("Carbon")); + + assertEquals(RowUtil.getFloat(data, 11), (float) 1.23); + i++; + } + reader.close(); + } catch (Throwable e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } finally { + try { + FileUtils.deleteDirectory(new File(path)); + } catch (IOException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + } + + @Test + public void testReadNextRowWithProjectionAndRowUtil() { + String path = "./carbondata"; + try { + FileUtils.deleteDirectory(new File(path)); + + Field[] fields = new Field[12]; + fields[0] = new Field("stringField", DataTypes.STRING); + fields[1] = new Field("shortField", DataTypes.SHORT); + fields[2] = new Field("intField", DataTypes.INT); + fields[3] = new Field("longField", DataTypes.LONG); + fields[4] = new Field("doubleField", DataTypes.DOUBLE); + fields[5] = new Field("boolField", DataTypes.BOOLEAN); + fields[6] = new Field("dateField", DataTypes.DATE); + fields[7] = new Field("timeField", DataTypes.TIMESTAMP); + fields[8] = new Field("decimalField", DataTypes.createDecimalType(8, 2)); + fields[9] = new Field("varcharField", DataTypes.VARCHAR); + fields[10] = new Field("arrayField", DataTypes.createArrayType(DataTypes.STRING)); + fields[11] = new Field("floatField", DataTypes.FLOAT); + Map<String, String> map = new HashMap<>(); + map.put("complex_delimiter_level_1", "#"); + CarbonWriter writer = CarbonWriter.builder() + .outputPath(path) + .withLoadOptions(map) + .withCsvInput(new Schema(fields)) + .writtenBy("CarbonReaderTest") + .build(); + + for (int i = 0; i < 10; i++) { + String[] row2 = new String[]{ + "robot" + (i % 10), + String.valueOf(i % 10000), + String.valueOf(i), + String.valueOf(Long.MAX_VALUE - i), + String.valueOf((double) i / 2), + String.valueOf(true), + "2019-03-02", + "2019-02-12 03:03:34", + "12.345", + "varchar", + "Hello#World#From#Carbon", + "1.23" + }; + writer.write(row2); + } + writer.close(); + + // Read data + CarbonReader reader = CarbonReader + .builder(path, "_temp") + .build(); + + int i = 0; + while (reader.hasNext()) { + Object[] data = (Object[]) reader.readNextRow(); + + assert (RowUtil.getString(data, 0).equals("robot" + i)); + assertEquals(RowUtil.getInt(data, 1), 17957); + assert (RowUtil.getVarchar(data, 3).equals("varchar")); + Object[] arr = RowUtil.getArray(data, 4); + assert (arr[0].equals("Hello")); + assert (arr[1].equals("World")); + assert (arr[2].equals("From")); + assert (arr[3].equals("Carbon")); + assertEquals(RowUtil.getShort(data, 5), i); + assertEquals(RowUtil.getInt(data, 6), i); + assertEquals(RowUtil.getLong(data, 7), Long.MAX_VALUE - i); + assertEquals(RowUtil.getDouble(data, 8), ((double) i) / 2); + assert (RowUtil.getBoolean(data, 9)); + assert (RowUtil.getDecimal(data, 10).equals("12.35")); + assertEquals(RowUtil.getFloat(data, 11), (float) 1.23); + i++; + } + reader.close(); + } catch (Throwable e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } finally { + try { + FileUtils.deleteDirectory(new File(path)); + } catch (IOException e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + } + }
