[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user asfgit closed the pull request at: https://github.com/apache/carbondata/pull/2899 ---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user xubo245 commented on a diff in the pull request: https://github.com/apache/carbondata/pull/2899#discussion_r241813858 --- Diff: store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java --- @@ -77,8 +77,9 @@ /** * Sets the output path of the writer builder + * --- End diff -- I think it's better for formatting the code comments/annotation ---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241811704
--- Diff: store/CSDK/src/CarbonWriter.cpp ---
@@ -98,6 +127,158 @@ void CarbonWriter::withHadoopConf(char *key, char
*value) {
carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
}
+void CarbonWriter::withTableProperty(char *key, char *value) {
+if (key == NULL) {
+throw std::runtime_error("key parameter can't be NULL.");
+}
+if (value == NULL) {
+throw std::runtime_error("value parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withTableProperty",
+
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withTableProperty");
+}
+jvalue args[2];
+args[0].l = jniEnv->NewStringUTF(key);
+args[1].l = jniEnv->NewStringUTF(value);
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
--- End diff --
ok
---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user KanakaKumar commented on a diff in the pull request: https://github.com/apache/carbondata/pull/2899#discussion_r241731133 --- Diff: store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java --- @@ -77,8 +77,9 @@ /** * Sets the output path of the writer builder + * --- End diff -- Can avoid the space changes in multiple places if not intentional. ---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user KanakaKumar commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241730265
--- Diff: store/CSDK/src/CarbonWriter.cpp ---
@@ -98,6 +127,158 @@ void CarbonWriter::withHadoopConf(char *key, char
*value) {
carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
}
+void CarbonWriter::withTableProperty(char *key, char *value) {
+if (key == NULL) {
+throw std::runtime_error("key parameter can't be NULL.");
+}
+if (value == NULL) {
+throw std::runtime_error("value parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withTableProperty",
+
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withTableProperty");
+}
+jvalue args[2];
+args[0].l = jniEnv->NewStringUTF(key);
+args[1].l = jniEnv->NewStringUTF(value);
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
--- End diff --
Java code may throw IllegalArgumentException for unsuported property. We
should add exception check after the method call
---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241630042
--- Diff: store/CSDK/test/main.cpp ---
@@ -645,6 +653,278 @@ bool testWriteData(JNIEnv *env, char *path, int argc,
char *argv[]) {
}
}
+void writeData(JNIEnv *env, CarbonWriter writer, int size, jclass
objClass, char *stringField, short shortField) {
+jobjectArray arr = env->NewObjectArray(size, objClass, 0);
+
+jobject jStringField = env->NewStringUTF(stringField);
+env->SetObjectArrayElement(arr, 0, jStringField);
+
+char ctrShort[10];
+gcvt(shortField % 1, 10, ctrShort);
+jobject jShortField = env->NewStringUTF(ctrShort);
+env->SetObjectArrayElement(arr, 1, jShortField);
+
+writer.write(arr);
+
+env->DeleteLocalRef(jStringField);
+env->DeleteLocalRef(jShortField);
+env->DeleteLocalRef(arr);
+}
+
+/**
+ * test WithLoadOption interface
+ *
+ * @param env jni env
+ * @param path file path
+ * @param argc argument counter
+ * @param argv argument vector
+ * @return true or throw exception
+ */
+bool testWithLoadOption(JNIEnv *env, char *path, int argc, char *argv[]) {
+
+char *jsonSchema =
"[{stringField:string},{shortField:short},{intField:int},{longField:long},{doubleField:double},{boolField:boolean},{dateField:date},{timeField:timestamp},{floatField:float},{arrayField:array}]";
+try {
+CarbonWriter writer;
+writer.builder(env);
+writer.outputPath(path);
+writer.withCsvInput(jsonSchema);
+writer.withLoadOption("complex_delimiter_level_1", "#");
+writer.writtenBy("CSDK");
+writer.taskNo(185);
+writer.withThreadSafe(1);
+writer.uniqueIdentifier(154991181400);
+writer.withBlockSize(1);
+writer.withBlockletSize(16);
+writer.enableLocalDictionary(true);
+writer.localDictionaryThreshold(1);
+if (argc > 3) {
+writer.withHadoopConf("fs.s3a.access.key", argv[1]);
+writer.withHadoopConf("fs.s3a.secret.key", argv[2]);
+writer.withHadoopConf("fs.s3a.endpoint", argv[3]);
+}
+writer.build();
+
+int rowNum = 7;
+int size = 10;
+long longValue = 0;
+double doubleValue = 0;
+float floatValue = 0;
+jclass objClass = env->FindClass("java/lang/String");
+for (int i = 0; i < rowNum; ++i) {
+jobjectArray arr = env->NewObjectArray(size, objClass, 0);
+char ctrInt[10];
+gcvt(i, 10, ctrInt);
+
+char a[15] = "robot";
+strcat(a, ctrInt);
+jobject stringField = env->NewStringUTF(a);
+env->SetObjectArrayElement(arr, 0, stringField);
+
+char ctrShort[10];
+gcvt(i % 1, 10, ctrShort);
+jobject shortField = env->NewStringUTF(ctrShort);
+env->SetObjectArrayElement(arr, 1, shortField);
+
+jobject intField = env->NewStringUTF(ctrInt);
+env->SetObjectArrayElement(arr, 2, intField);
+
+
+char ctrLong[10];
+gcvt(longValue, 10, ctrLong);
+longValue = longValue + 2;
+jobject longField = env->NewStringUTF(ctrLong);
+env->SetObjectArrayElement(arr, 3, longField);
+
+char ctrDouble[10];
+gcvt(doubleValue, 10, ctrDouble);
+doubleValue = doubleValue + 2;
+jobject doubleField = env->NewStringUTF(ctrDouble);
+env->SetObjectArrayElement(arr, 4, doubleField);
+
+jobject boolField = env->NewStringUTF("true");
+env->SetObjectArrayElement(arr, 5, boolField);
+
+jobject dateField = env->NewStringUTF(" 2019-03-02");
+env->SetObjectArrayElement(arr, 6, dateField);
+
+jobject timeField = env->NewStringUTF("2019-02-12 03:03:34");
+env->SetObjectArrayElement(arr, 7, timeField);
+
+char ctrFloat[10];
+gcvt(floatValue, 10, ctrFloat);
+floatValue = floatValue + 2;
+jobject floatField = env->NewStringUTF(ctrFloat);
+env->SetObjectArrayElement(arr, 8, floatField);
+
+jobject arrayField =
env->NewStringUTF("Hello#World#From#Carbon");
+env->SetObjectArrayElement(arr, 9, arrayField);
+
+
+writer.write(arr);
+
+env->DeleteLocalRef(stringField);
+env->DeleteLocalRef(shortField);
+
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241629538
--- Diff: store/CSDK/test/main.cpp ---
@@ -645,6 +653,278 @@ bool testWriteData(JNIEnv *env, char *path, int argc,
char *argv[]) {
}
}
+void writeData(JNIEnv *env, CarbonWriter writer, int size, jclass
objClass, char *stringField, short shortField) {
+jobjectArray arr = env->NewObjectArray(size, objClass, 0);
+
+jobject jStringField = env->NewStringUTF(stringField);
+env->SetObjectArrayElement(arr, 0, jStringField);
+
+char ctrShort[10];
+gcvt(shortField % 1, 10, ctrShort);
+jobject jShortField = env->NewStringUTF(ctrShort);
+env->SetObjectArrayElement(arr, 1, jShortField);
+
+writer.write(arr);
+
+env->DeleteLocalRef(jStringField);
+env->DeleteLocalRef(jShortField);
+env->DeleteLocalRef(arr);
+}
+
+/**
+ * test WithLoadOption interface
+ *
+ * @param env jni env
+ * @param path file path
+ * @param argc argument counter
+ * @param argv argument vector
+ * @return true or throw exception
+ */
+bool testWithLoadOption(JNIEnv *env, char *path, int argc, char *argv[]) {
+
+char *jsonSchema =
"[{stringField:string},{shortField:short},{intField:int},{longField:long},{doubleField:double},{boolField:boolean},{dateField:date},{timeField:timestamp},{floatField:float},{arrayField:array}]";
+try {
+CarbonWriter writer;
+writer.builder(env);
--- End diff --
ok, move it to testWriteData
---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241627445
--- Diff:
store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
---
@@ -543,4 +543,42 @@ public void testWritingAndReadingArrayOfFloatAndByte()
throws IOException {
}
}
+ @Test
+ public void testWithTableProperties() throws IOException {
--- End diff --
valid the table withTableProperty sort_by, input disorder dataï¼ after set
sort_by by withTableProperty, the data will become order.
---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241627223
--- Diff:
store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
---
@@ -199,6 +203,22 @@ public CarbonWriterBuilder withLoadOptions(Map options) {
return this;
}
+ /**
+ * To support the load options for sdk writer
+ *
+ * @param key the key of load option
+ * @param value the value of load option
+ * @return updated CarbonWriterBuilder object
+ */
+ public CarbonWriterBuilder withLoadOption(String key, String value) {
+Objects.requireNonNull(key, "key of table properties should not be
null");
--- End diff --
ok, done
---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241627031
--- Diff: store/CSDK/src/CarbonWriter.cpp ---
@@ -58,6 +58,35 @@ void CarbonWriter::outputPath(char *path) {
carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
}
+void CarbonWriter::sortBy(int argc, char **argv) {
+if (argc < 0) {
+throw std::runtime_error("argc parameter can't be negative.");
+}
+if (argv == NULL) {
+throw std::runtime_error("argv parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonReaderBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonReaderBuilderClass,
"sortBy",
+
"([Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java: sortBy");
+}
+jclass objectArrayClass = jniEnv->FindClass("Ljava/lang/String;");
+if (objectArrayClass == NULL) {
+throw std::runtime_error("Can't find the class in java:
java/lang/String");
+}
+jobjectArray array = jniEnv->NewObjectArray(argc, objectArrayClass,
NULL);
+for (int i = 0; i < argc; ++i) {
+jstring value = jniEnv->NewStringUTF(argv[i]);
+jniEnv->SetObjectArrayElement(array, i, value);
+}
+
+jvalue args[1];
+args[0].l = array;
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
--- End diff --
sortBy in CPP is void, but sortBy in Java isn't void, we should change the
carbonWriterBuilderObject in class
---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241626623
--- Diff: store/CSDK/src/CarbonWriter.cpp ---
@@ -98,6 +127,158 @@ void CarbonWriter::withHadoopConf(char *key, char
*value) {
carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
}
+void CarbonWriter::withTableProperty(char *key, char *value) {
+if (key == NULL) {
+throw std::runtime_error("key parameter can't be NULL.");
+}
+if (value == NULL) {
+throw std::runtime_error("value parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withTableProperty",
+
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withTableProperty");
+}
+jvalue args[2];
+args[0].l = jniEnv->NewStringUTF(key);
+args[1].l = jniEnv->NewStringUTF(value);
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::withLoadOption(char *key, char *value) {
+if (key == NULL) {
+throw std::runtime_error("key parameter can't be NULL.");
+}
+if (value == NULL) {
+throw std::runtime_error("value parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withLoadOption",
+
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withLoadOption");
+}
+jvalue args[2];
+args[0].l = jniEnv->NewStringUTF(key);
+args[1].l = jniEnv->NewStringUTF(value);
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::taskNo(long taskNo) {
+if (taskNo < 0) {
+throw std::runtime_error("taskNo parameter can't be negative.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"taskNo",
+"(J)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java: taskNo");
+}
+jvalue args[2];
+args[0].j = taskNo;
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::uniqueIdentifier(long timestamp) {
+if (timestamp < 1) {
+throw std::runtime_error("timestamp parameter can't be negative.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"uniqueIdentifier",
+"(J)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
uniqueIdentifier");
+}
+jvalue args[2];
+args[0].j = timestamp;
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::withThreadSafe(short numOfThreads) {
+if (numOfThreads < 1) {
+throw std::runtime_error("numOfThreads parameter can't be
negative.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withThreadSafe",
+"(S)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withThreadSafe");
+}
+jvalue args[2];
+args[0].s = numOfThreads;
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::withBlockSize(int blockSize) {
+if (blockSize < 1) {
+throw std::runtime_error("blockSize parameter should be positive
number.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241626607
--- Diff: store/CSDK/src/CarbonWriter.cpp ---
@@ -98,6 +127,158 @@ void CarbonWriter::withHadoopConf(char *key, char
*value) {
carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
}
+void CarbonWriter::withTableProperty(char *key, char *value) {
+if (key == NULL) {
+throw std::runtime_error("key parameter can't be NULL.");
+}
+if (value == NULL) {
+throw std::runtime_error("value parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withTableProperty",
+
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withTableProperty");
+}
+jvalue args[2];
+args[0].l = jniEnv->NewStringUTF(key);
+args[1].l = jniEnv->NewStringUTF(value);
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::withLoadOption(char *key, char *value) {
+if (key == NULL) {
+throw std::runtime_error("key parameter can't be NULL.");
+}
+if (value == NULL) {
+throw std::runtime_error("value parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withLoadOption",
+
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withLoadOption");
+}
+jvalue args[2];
+args[0].l = jniEnv->NewStringUTF(key);
+args[1].l = jniEnv->NewStringUTF(value);
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::taskNo(long taskNo) {
+if (taskNo < 0) {
+throw std::runtime_error("taskNo parameter can't be negative.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"taskNo",
+"(J)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java: taskNo");
+}
+jvalue args[2];
+args[0].j = taskNo;
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::uniqueIdentifier(long timestamp) {
+if (timestamp < 1) {
+throw std::runtime_error("timestamp parameter can't be negative.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"uniqueIdentifier",
+"(J)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
uniqueIdentifier");
+}
+jvalue args[2];
+args[0].j = timestamp;
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::withThreadSafe(short numOfThreads) {
+if (numOfThreads < 1) {
+throw std::runtime_error("numOfThreads parameter can't be
negative.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withThreadSafe",
+"(S)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withThreadSafe");
+}
+jvalue args[2];
--- End diff --
yes, done
---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241626594
--- Diff: store/CSDK/src/CarbonWriter.cpp ---
@@ -98,6 +127,158 @@ void CarbonWriter::withHadoopConf(char *key, char
*value) {
carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
}
+void CarbonWriter::withTableProperty(char *key, char *value) {
+if (key == NULL) {
+throw std::runtime_error("key parameter can't be NULL.");
+}
+if (value == NULL) {
+throw std::runtime_error("value parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withTableProperty",
+
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withTableProperty");
+}
+jvalue args[2];
+args[0].l = jniEnv->NewStringUTF(key);
+args[1].l = jniEnv->NewStringUTF(value);
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::withLoadOption(char *key, char *value) {
+if (key == NULL) {
+throw std::runtime_error("key parameter can't be NULL.");
+}
+if (value == NULL) {
+throw std::runtime_error("value parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withLoadOption",
+
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withLoadOption");
+}
+jvalue args[2];
+args[0].l = jniEnv->NewStringUTF(key);
+args[1].l = jniEnv->NewStringUTF(value);
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::taskNo(long taskNo) {
+if (taskNo < 0) {
+throw std::runtime_error("taskNo parameter can't be negative.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"taskNo",
+"(J)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java: taskNo");
+}
+jvalue args[2];
+args[0].j = taskNo;
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::uniqueIdentifier(long timestamp) {
+if (timestamp < 1) {
+throw std::runtime_error("timestamp parameter can't be negative.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"uniqueIdentifier",
+"(J)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
uniqueIdentifier");
+}
+jvalue args[2];
--- End diff --
yes, done
---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241626570
--- Diff: store/CSDK/src/CarbonWriter.cpp ---
@@ -98,6 +127,158 @@ void CarbonWriter::withHadoopConf(char *key, char
*value) {
carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
}
+void CarbonWriter::withTableProperty(char *key, char *value) {
+if (key == NULL) {
+throw std::runtime_error("key parameter can't be NULL.");
+}
+if (value == NULL) {
+throw std::runtime_error("value parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withTableProperty",
+
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withTableProperty");
+}
+jvalue args[2];
+args[0].l = jniEnv->NewStringUTF(key);
+args[1].l = jniEnv->NewStringUTF(value);
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::withLoadOption(char *key, char *value) {
+if (key == NULL) {
+throw std::runtime_error("key parameter can't be NULL.");
+}
+if (value == NULL) {
+throw std::runtime_error("value parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withLoadOption",
+
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withLoadOption");
+}
+jvalue args[2];
+args[0].l = jniEnv->NewStringUTF(key);
+args[1].l = jniEnv->NewStringUTF(value);
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::taskNo(long taskNo) {
+if (taskNo < 0) {
+throw std::runtime_error("taskNo parameter can't be negative.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"taskNo",
+"(J)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java: taskNo");
+}
+jvalue args[2];
--- End diff --
yes, done
---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241626357
--- Diff: store/CSDK/src/CarbonWriter.cpp ---
@@ -58,6 +58,35 @@ void CarbonWriter::outputPath(char *path) {
carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
}
+void CarbonWriter::sortBy(int argc, char **argv) {
+if (argc < 0) {
+throw std::runtime_error("argc parameter can't be negative.");
+}
+if (argv == NULL) {
+throw std::runtime_error("argv parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonReaderBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
--- End diff --
done
---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user xubo245 commented on a diff in the pull request: https://github.com/apache/carbondata/pull/2899#discussion_r241625910 --- Diff: docs/csdk-guide.md --- @@ -214,6 +226,122 @@ release the memory and destroy JVM. void withHadoopConf(char *key, char *value); ``` +<<< HEAD +=== +``` + /** + * To support the table properties for writer + * + * @param key properties key + * @param value properties value + */ +void withTableProperty(char *key, char *value); +``` + +``` +/** + * To support the load options for C++ sdk writer + * + * @param options key,value pair of load options. + * supported keys values are + * a. bad_records_logger_enable -- true (write into separate logs), false + * b. bad_records_action -- FAIL, FORCE, IGNORE, REDIRECT + * c. bad_record_path -- path + * d. dateformat -- same as JAVA SimpleDateFormat + * e. timestampformat -- same as JAVA SimpleDateFormat + * f. complex_delimiter_level_1 -- value to Split the complexTypeData + * g. complex_delimiter_level_2 -- value to Split the nested complexTypeData + * h. quotechar + * i. escapechar + * + * Default values are as follows. + * + * a. bad_records_logger_enable -- "false" + * b. bad_records_action -- "FAIL" + * c. bad_record_path -- "" + * d. dateformat -- "" , uses from carbon.properties file + * e. timestampformat -- "", uses from carbon.properties file + * f. complex_delimiter_level_1 -- "$" + * g. complex_delimiter_level_2 -- ":" + * h. quotechar -- "\"" + * i. escapechar -- "\\" + * + * @return updated CarbonWriterBuilder + */ +void withLoadOption(char *key, char *value); +``` + +``` +/** + * sets the taskNo for the writer. CSDKs concurrently running + * will set taskNo in order to avoid conflicts in file's name during write. + * + * @param taskNo is the TaskNo user wants to specify. + * by default it is system time in nano seconds. + */ +void taskNo(long taskNo); +``` + +``` +/** + * to set the timestamp in the carbondata and carbonindex index files + * + * @param timestamp is a timestamp to be used in the carbondata and carbonindex index files. + * By default set to zero. + * @return updated CarbonWriterBuilder + */ +void uniqueIdentifier(long timestamp); +``` + +``` +/** + * To make c++ sdk writer thread safe. + * + * @param numOfThreads should number of threads in which writer is called in multi-thread scenario + * default C++ sdk writer is not thread safe. + * can use one writer instance in one thread only. + */ +void withThreadSafe(short numOfThreads) ; +``` + +``` +/** + * To set the carbondata file size in MB between 1MB-2048MB + * + * @param blockSize is size in MB between 1MB to 2048 MB + * default value is 1024 MB + */ +void withBlockSize(int blockSize); +``` + +``` +/** + * To set the blocklet size of CarbonData file + * + * @param blockletSize is blocklet size in MB + *default value is 64 MB + * @return updated CarbonWriterBuilder + */ +void withBlockletSize(int blockletSize); +``` + +``` +/** + * @param localDictionaryThreshold is localDictionaryThreshold, default is 1 + * @return updated CarbonWriterBuilder + */ +void localDictionaryThreshold(int localDictionaryThreshold); +``` + +``` +/** + * @param enableLocalDictionary enable local dictionary, default is false + * @return updated CarbonWriterBuilder + */ +void enableLocalDictionary(bool enableLocalDictionary); +``` + +>>> aebd066bc... [CARBONDATA-3073] Support configure TableProperties,withLoadOption etc. interface in carbon writer of C++ SDK --- End diff -- ok, done ---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user xubo245 commented on a diff in the pull request: https://github.com/apache/carbondata/pull/2899#discussion_r241625744 --- Diff: docs/csdk-guide.md --- @@ -214,6 +226,122 @@ release the memory and destroy JVM. void withHadoopConf(char *key, char *value); ``` +<<< HEAD --- End diff -- sorry, rebase error, rebase many times before ---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241334135
--- Diff: store/CSDK/test/main.cpp ---
@@ -645,6 +653,278 @@ bool testWriteData(JNIEnv *env, char *path, int argc,
char *argv[]) {
}
}
+void writeData(JNIEnv *env, CarbonWriter writer, int size, jclass
objClass, char *stringField, short shortField) {
+jobjectArray arr = env->NewObjectArray(size, objClass, 0);
+
+jobject jStringField = env->NewStringUTF(stringField);
+env->SetObjectArrayElement(arr, 0, jStringField);
+
+char ctrShort[10];
+gcvt(shortField % 1, 10, ctrShort);
+jobject jShortField = env->NewStringUTF(ctrShort);
+env->SetObjectArrayElement(arr, 1, jShortField);
+
+writer.write(arr);
+
+env->DeleteLocalRef(jStringField);
+env->DeleteLocalRef(jShortField);
+env->DeleteLocalRef(arr);
+}
+
+/**
+ * test WithLoadOption interface
+ *
+ * @param env jni env
+ * @param path file path
+ * @param argc argument counter
+ * @param argv argument vector
+ * @return true or throw exception
+ */
+bool testWithLoadOption(JNIEnv *env, char *path, int argc, char *argv[]) {
+
+char *jsonSchema =
"[{stringField:string},{shortField:short},{intField:int},{longField:long},{doubleField:double},{boolField:boolean},{dateField:date},{timeField:timestamp},{floatField:float},{arrayField:array}]";
+try {
+CarbonWriter writer;
+writer.builder(env);
+writer.outputPath(path);
+writer.withCsvInput(jsonSchema);
+writer.withLoadOption("complex_delimiter_level_1", "#");
+writer.writtenBy("CSDK");
+writer.taskNo(185);
+writer.withThreadSafe(1);
+writer.uniqueIdentifier(154991181400);
+writer.withBlockSize(1);
+writer.withBlockletSize(16);
+writer.enableLocalDictionary(true);
+writer.localDictionaryThreshold(1);
+if (argc > 3) {
+writer.withHadoopConf("fs.s3a.access.key", argv[1]);
+writer.withHadoopConf("fs.s3a.secret.key", argv[2]);
+writer.withHadoopConf("fs.s3a.endpoint", argv[3]);
+}
+writer.build();
+
+int rowNum = 7;
+int size = 10;
+long longValue = 0;
+double doubleValue = 0;
+float floatValue = 0;
+jclass objClass = env->FindClass("java/lang/String");
+for (int i = 0; i < rowNum; ++i) {
+jobjectArray arr = env->NewObjectArray(size, objClass, 0);
+char ctrInt[10];
+gcvt(i, 10, ctrInt);
+
+char a[15] = "robot";
+strcat(a, ctrInt);
+jobject stringField = env->NewStringUTF(a);
+env->SetObjectArrayElement(arr, 0, stringField);
+
+char ctrShort[10];
+gcvt(i % 1, 10, ctrShort);
+jobject shortField = env->NewStringUTF(ctrShort);
+env->SetObjectArrayElement(arr, 1, shortField);
+
+jobject intField = env->NewStringUTF(ctrInt);
+env->SetObjectArrayElement(arr, 2, intField);
+
+
+char ctrLong[10];
+gcvt(longValue, 10, ctrLong);
+longValue = longValue + 2;
+jobject longField = env->NewStringUTF(ctrLong);
+env->SetObjectArrayElement(arr, 3, longField);
+
+char ctrDouble[10];
+gcvt(doubleValue, 10, ctrDouble);
+doubleValue = doubleValue + 2;
+jobject doubleField = env->NewStringUTF(ctrDouble);
+env->SetObjectArrayElement(arr, 4, doubleField);
+
+jobject boolField = env->NewStringUTF("true");
+env->SetObjectArrayElement(arr, 5, boolField);
+
+jobject dateField = env->NewStringUTF(" 2019-03-02");
+env->SetObjectArrayElement(arr, 6, dateField);
+
+jobject timeField = env->NewStringUTF("2019-02-12 03:03:34");
+env->SetObjectArrayElement(arr, 7, timeField);
+
+char ctrFloat[10];
+gcvt(floatValue, 10, ctrFloat);
+floatValue = floatValue + 2;
+jobject floatField = env->NewStringUTF(ctrFloat);
+env->SetObjectArrayElement(arr, 8, floatField);
+
+jobject arrayField =
env->NewStringUTF("Hello#World#From#Carbon");
+env->SetObjectArrayElement(arr, 9, arrayField);
+
+
+writer.write(arr);
+
+env->DeleteLocalRef(stringField);
+env->DeleteLocalRef(shortField);
+
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241333995
--- Diff: store/CSDK/test/main.cpp ---
@@ -645,6 +653,278 @@ bool testWriteData(JNIEnv *env, char *path, int argc,
char *argv[]) {
}
}
+void writeData(JNIEnv *env, CarbonWriter writer, int size, jclass
objClass, char *stringField, short shortField) {
+jobjectArray arr = env->NewObjectArray(size, objClass, 0);
+
+jobject jStringField = env->NewStringUTF(stringField);
+env->SetObjectArrayElement(arr, 0, jStringField);
+
+char ctrShort[10];
+gcvt(shortField % 1, 10, ctrShort);
+jobject jShortField = env->NewStringUTF(ctrShort);
+env->SetObjectArrayElement(arr, 1, jShortField);
+
+writer.write(arr);
+
+env->DeleteLocalRef(jStringField);
+env->DeleteLocalRef(jShortField);
+env->DeleteLocalRef(arr);
+}
+
+/**
+ * test WithLoadOption interface
+ *
+ * @param env jni env
+ * @param path file path
+ * @param argc argument counter
+ * @param argv argument vector
+ * @return true or throw exception
+ */
+bool testWithLoadOption(JNIEnv *env, char *path, int argc, char *argv[]) {
+
+char *jsonSchema =
"[{stringField:string},{shortField:short},{intField:int},{longField:long},{doubleField:double},{boolField:boolean},{dateField:date},{timeField:timestamp},{floatField:float},{arrayField:array}]";
+try {
+CarbonWriter writer;
+writer.builder(env);
--- End diff --
Can add these load options to existing writer test case, no need to add new
---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241328653
--- Diff:
store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
---
@@ -543,4 +543,42 @@ public void testWritingAndReadingArrayOfFloatAndByte()
throws IOException {
}
}
+ @Test
+ public void testWithTableProperties() throws IOException {
--- End diff --
what's the use of this test case ? how would you validate table property is
working or not ?
---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241327234
--- Diff:
store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
---
@@ -199,6 +203,22 @@ public CarbonWriterBuilder withLoadOptions(Map options) {
return this;
}
+ /**
+ * To support the load options for sdk writer
+ *
+ * @param key the key of load option
+ * @param value the value of load option
+ * @return updated CarbonWriterBuilder object
+ */
+ public CarbonWriterBuilder withLoadOption(String key, String value) {
+Objects.requireNonNull(key, "key of table properties should not be
null");
--- End diff --
this is load options, not table properties
---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241322296
--- Diff: store/CSDK/src/CarbonWriter.cpp ---
@@ -58,6 +58,35 @@ void CarbonWriter::outputPath(char *path) {
carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
}
+void CarbonWriter::sortBy(int argc, char **argv) {
+if (argc < 0) {
+throw std::runtime_error("argc parameter can't be negative.");
+}
+if (argv == NULL) {
+throw std::runtime_error("argv parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonReaderBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonReaderBuilderClass,
"sortBy",
+
"([Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java: sortBy");
+}
+jclass objectArrayClass = jniEnv->FindClass("Ljava/lang/String;");
+if (objectArrayClass == NULL) {
+throw std::runtime_error("Can't find the class in java:
java/lang/String");
+}
+jobjectArray array = jniEnv->NewObjectArray(argc, objectArrayClass,
NULL);
+for (int i = 0; i < argc; ++i) {
+jstring value = jniEnv->NewStringUTF(argv[i]);
+jniEnv->SetObjectArrayElement(array, i, value);
+}
+
+jvalue args[1];
+args[0].l = array;
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
--- End diff --
can this be modified to
(void) jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
no need to collect return value from java API as it return type of cpp
method is void.
check same for all the new void API added
---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241321530
--- Diff: store/CSDK/src/CarbonWriter.cpp ---
@@ -98,6 +127,158 @@ void CarbonWriter::withHadoopConf(char *key, char
*value) {
carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
}
+void CarbonWriter::withTableProperty(char *key, char *value) {
+if (key == NULL) {
+throw std::runtime_error("key parameter can't be NULL.");
+}
+if (value == NULL) {
+throw std::runtime_error("value parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withTableProperty",
+
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withTableProperty");
+}
+jvalue args[2];
+args[0].l = jniEnv->NewStringUTF(key);
+args[1].l = jniEnv->NewStringUTF(value);
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::withLoadOption(char *key, char *value) {
+if (key == NULL) {
+throw std::runtime_error("key parameter can't be NULL.");
+}
+if (value == NULL) {
+throw std::runtime_error("value parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withLoadOption",
+
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withLoadOption");
+}
+jvalue args[2];
+args[0].l = jniEnv->NewStringUTF(key);
+args[1].l = jniEnv->NewStringUTF(value);
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::taskNo(long taskNo) {
+if (taskNo < 0) {
+throw std::runtime_error("taskNo parameter can't be negative.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"taskNo",
+"(J)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java: taskNo");
+}
+jvalue args[2];
+args[0].j = taskNo;
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::uniqueIdentifier(long timestamp) {
+if (timestamp < 1) {
+throw std::runtime_error("timestamp parameter can't be negative.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"uniqueIdentifier",
+"(J)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
uniqueIdentifier");
+}
+jvalue args[2];
+args[0].j = timestamp;
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::withThreadSafe(short numOfThreads) {
+if (numOfThreads < 1) {
+throw std::runtime_error("numOfThreads parameter can't be
negative.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withThreadSafe",
+"(S)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withThreadSafe");
+}
+jvalue args[2];
+args[0].s = numOfThreads;
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::withBlockSize(int blockSize) {
+if (blockSize < 1) {
+throw std::runtime_error("blockSize parameter should be positive
number.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241320959
--- Diff: store/CSDK/src/CarbonWriter.cpp ---
@@ -98,6 +127,158 @@ void CarbonWriter::withHadoopConf(char *key, char
*value) {
carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
}
+void CarbonWriter::withTableProperty(char *key, char *value) {
+if (key == NULL) {
+throw std::runtime_error("key parameter can't be NULL.");
+}
+if (value == NULL) {
+throw std::runtime_error("value parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withTableProperty",
+
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withTableProperty");
+}
+jvalue args[2];
+args[0].l = jniEnv->NewStringUTF(key);
+args[1].l = jniEnv->NewStringUTF(value);
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::withLoadOption(char *key, char *value) {
+if (key == NULL) {
+throw std::runtime_error("key parameter can't be NULL.");
+}
+if (value == NULL) {
+throw std::runtime_error("value parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withLoadOption",
+
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withLoadOption");
+}
+jvalue args[2];
+args[0].l = jniEnv->NewStringUTF(key);
+args[1].l = jniEnv->NewStringUTF(value);
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::taskNo(long taskNo) {
+if (taskNo < 0) {
+throw std::runtime_error("taskNo parameter can't be negative.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"taskNo",
+"(J)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java: taskNo");
+}
+jvalue args[2];
+args[0].j = taskNo;
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::uniqueIdentifier(long timestamp) {
+if (timestamp < 1) {
+throw std::runtime_error("timestamp parameter can't be negative.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"uniqueIdentifier",
+"(J)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
uniqueIdentifier");
+}
+jvalue args[2];
+args[0].j = timestamp;
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::withThreadSafe(short numOfThreads) {
+if (numOfThreads < 1) {
+throw std::runtime_error("numOfThreads parameter can't be
negative.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withThreadSafe",
+"(S)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withThreadSafe");
+}
+jvalue args[2];
--- End diff --
same as above
---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241318527
--- Diff: store/CSDK/src/CarbonWriter.cpp ---
@@ -98,6 +127,158 @@ void CarbonWriter::withHadoopConf(char *key, char
*value) {
carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
}
+void CarbonWriter::withTableProperty(char *key, char *value) {
+if (key == NULL) {
+throw std::runtime_error("key parameter can't be NULL.");
+}
+if (value == NULL) {
+throw std::runtime_error("value parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withTableProperty",
+
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withTableProperty");
+}
+jvalue args[2];
+args[0].l = jniEnv->NewStringUTF(key);
+args[1].l = jniEnv->NewStringUTF(value);
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::withLoadOption(char *key, char *value) {
+if (key == NULL) {
+throw std::runtime_error("key parameter can't be NULL.");
+}
+if (value == NULL) {
+throw std::runtime_error("value parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withLoadOption",
+
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withLoadOption");
+}
+jvalue args[2];
+args[0].l = jniEnv->NewStringUTF(key);
+args[1].l = jniEnv->NewStringUTF(value);
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::taskNo(long taskNo) {
+if (taskNo < 0) {
+throw std::runtime_error("taskNo parameter can't be negative.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"taskNo",
+"(J)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java: taskNo");
+}
+jvalue args[2];
+args[0].j = taskNo;
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::uniqueIdentifier(long timestamp) {
+if (timestamp < 1) {
+throw std::runtime_error("timestamp parameter can't be negative.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"uniqueIdentifier",
+"(J)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
uniqueIdentifier");
+}
+jvalue args[2];
--- End diff --
should be size 1 ?
---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241318351
--- Diff: store/CSDK/src/CarbonWriter.cpp ---
@@ -98,6 +127,158 @@ void CarbonWriter::withHadoopConf(char *key, char
*value) {
carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
}
+void CarbonWriter::withTableProperty(char *key, char *value) {
+if (key == NULL) {
+throw std::runtime_error("key parameter can't be NULL.");
+}
+if (value == NULL) {
+throw std::runtime_error("value parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withTableProperty",
+
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withTableProperty");
+}
+jvalue args[2];
+args[0].l = jniEnv->NewStringUTF(key);
+args[1].l = jniEnv->NewStringUTF(value);
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::withLoadOption(char *key, char *value) {
+if (key == NULL) {
+throw std::runtime_error("key parameter can't be NULL.");
+}
+if (value == NULL) {
+throw std::runtime_error("value parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"withLoadOption",
+
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java:
withLoadOption");
+}
+jvalue args[2];
+args[0].l = jniEnv->NewStringUTF(key);
+args[1].l = jniEnv->NewStringUTF(value);
+carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
+}
+
+void CarbonWriter::taskNo(long taskNo) {
+if (taskNo < 0) {
+throw std::runtime_error("taskNo parameter can't be negative.");
+}
+checkBuilder();
+jclass carbonWriterBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
+jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass,
"taskNo",
+"(J)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
+if (methodID == NULL) {
+throw std::runtime_error("Can't find the method in java: taskNo");
+}
+jvalue args[2];
--- End diff --
should be size 1 ?
---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2899#discussion_r241315209
--- Diff: store/CSDK/src/CarbonWriter.cpp ---
@@ -58,6 +58,35 @@ void CarbonWriter::outputPath(char *path) {
carbonWriterBuilderObject =
jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
}
+void CarbonWriter::sortBy(int argc, char **argv) {
+if (argc < 0) {
+throw std::runtime_error("argc parameter can't be negative.");
+}
+if (argv == NULL) {
+throw std::runtime_error("argv parameter can't be NULL.");
+}
+checkBuilder();
+jclass carbonReaderBuilderClass =
jniEnv->GetObjectClass(carbonWriterBuilderObject);
--- End diff --
it is writerBuilder, not readerBuilder
---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user ajantha-bhat commented on a diff in the pull request: https://github.com/apache/carbondata/pull/2899#discussion_r241313087 --- Diff: docs/csdk-guide.md --- @@ -214,6 +226,122 @@ release the memory and destroy JVM. void withHadoopConf(char *key, char *value); ``` +<<< HEAD +=== +``` + /** + * To support the table properties for writer + * + * @param key properties key + * @param value properties value + */ +void withTableProperty(char *key, char *value); +``` + +``` +/** + * To support the load options for C++ sdk writer + * + * @param options key,value pair of load options. + * supported keys values are + * a. bad_records_logger_enable -- true (write into separate logs), false + * b. bad_records_action -- FAIL, FORCE, IGNORE, REDIRECT + * c. bad_record_path -- path + * d. dateformat -- same as JAVA SimpleDateFormat + * e. timestampformat -- same as JAVA SimpleDateFormat + * f. complex_delimiter_level_1 -- value to Split the complexTypeData + * g. complex_delimiter_level_2 -- value to Split the nested complexTypeData + * h. quotechar + * i. escapechar + * + * Default values are as follows. + * + * a. bad_records_logger_enable -- "false" + * b. bad_records_action -- "FAIL" + * c. bad_record_path -- "" + * d. dateformat -- "" , uses from carbon.properties file + * e. timestampformat -- "", uses from carbon.properties file + * f. complex_delimiter_level_1 -- "$" + * g. complex_delimiter_level_2 -- ":" + * h. quotechar -- "\"" + * i. escapechar -- "\\" + * + * @return updated CarbonWriterBuilder + */ +void withLoadOption(char *key, char *value); +``` + +``` +/** + * sets the taskNo for the writer. CSDKs concurrently running + * will set taskNo in order to avoid conflicts in file's name during write. + * + * @param taskNo is the TaskNo user wants to specify. + * by default it is system time in nano seconds. + */ +void taskNo(long taskNo); +``` + +``` +/** + * to set the timestamp in the carbondata and carbonindex index files + * + * @param timestamp is a timestamp to be used in the carbondata and carbonindex index files. + * By default set to zero. + * @return updated CarbonWriterBuilder + */ +void uniqueIdentifier(long timestamp); +``` + +``` +/** + * To make c++ sdk writer thread safe. + * + * @param numOfThreads should number of threads in which writer is called in multi-thread scenario + * default C++ sdk writer is not thread safe. + * can use one writer instance in one thread only. + */ +void withThreadSafe(short numOfThreads) ; +``` + +``` +/** + * To set the carbondata file size in MB between 1MB-2048MB + * + * @param blockSize is size in MB between 1MB to 2048 MB + * default value is 1024 MB + */ +void withBlockSize(int blockSize); +``` + +``` +/** + * To set the blocklet size of CarbonData file + * + * @param blockletSize is blocklet size in MB + *default value is 64 MB + * @return updated CarbonWriterBuilder + */ +void withBlockletSize(int blockletSize); +``` + +``` +/** + * @param localDictionaryThreshold is localDictionaryThreshold, default is 1 + * @return updated CarbonWriterBuilder + */ +void localDictionaryThreshold(int localDictionaryThreshold); +``` + +``` +/** + * @param enableLocalDictionary enable local dictionary, default is false + * @return updated CarbonWriterBuilder + */ +void enableLocalDictionary(bool enableLocalDictionary); +``` + +>>> aebd066bc... [CARBONDATA-3073] Support configure TableProperties,withLoadOption etc. interface in carbon writer of C++ SDK --- End diff -- same as above ---
[GitHub] carbondata pull request #2899: [CARBONDATA-3073][CARBONDATA-3044] Support co...
Github user ajantha-bhat commented on a diff in the pull request: https://github.com/apache/carbondata/pull/2899#discussion_r241311890 --- Diff: docs/csdk-guide.md --- @@ -214,6 +226,122 @@ release the memory and destroy JVM. void withHadoopConf(char *key, char *value); ``` +<<< HEAD --- End diff -- what's this? problem while resolving a conflict? please check and redo this file. ---
