EricJoy2048 commented on code in PR #3375: URL: https://github.com/apache/incubator-seatunnel/pull/3375#discussion_r1018857653
########## docs/en/connector-v2/sink/Hive.md: ########## @@ -24,15 +24,11 @@ By default, we use 2PC commit to ensure `exactly-once` ## Options -| name | type | required | default value | -|-----------------------| ------ |---------------------------------------------| ------------------------------------------------------------- | -| table_name | string | yes | - | -| metastore_uri | string | yes | - | -| partition_by | array | required if hive sink table have partitions | - | Review Comment: Why remove this parameter? ########## seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseSinkConfig.java: ########## @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.file.config; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.common.utils.DateTimeUtils; +import org.apache.seatunnel.common.utils.DateUtils; +import org.apache.seatunnel.common.utils.TimeUtils; + +import java.util.List; + +public class BaseSinkConfig { + public static final String SEATUNNEL = "seatunnel"; + public static final String NON_PARTITION = "NON_PARTITION"; + public static final String TRANSACTION_ID_SPLIT = "_"; + public static final String TRANSACTION_EXPRESSION = "transactionId"; + public static final Option<String> COMPRESS_CODEC = Options.key("compress_codec") Review Comment: Please add this parameter to sink doc. ########## seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseSinkConfig.java: ########## @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.file.config; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.common.utils.DateTimeUtils; +import org.apache.seatunnel.common.utils.DateUtils; +import org.apache.seatunnel.common.utils.TimeUtils; + +import java.util.List; + +public class BaseSinkConfig { + public static final String SEATUNNEL = "seatunnel"; + public static final String NON_PARTITION = "NON_PARTITION"; + public static final String TRANSACTION_ID_SPLIT = "_"; + public static final String TRANSACTION_EXPRESSION = "transactionId"; + public static final Option<String> COMPRESS_CODEC = Options.key("compress_codec") + .stringType() + .noDefaultValue() + .withDescription("Compression codec"); + public static final Option<String> DATE_FORMAT = Options.key("date_format") Review Comment: same as above. ########## seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseSinkConfig.java: ########## @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.file.config; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.common.utils.DateTimeUtils; +import org.apache.seatunnel.common.utils.DateUtils; +import org.apache.seatunnel.common.utils.TimeUtils; + +import java.util.List; + +public class BaseSinkConfig { + public static final String SEATUNNEL = "seatunnel"; + public static final String NON_PARTITION = "NON_PARTITION"; + public static final String TRANSACTION_ID_SPLIT = "_"; + public static final String TRANSACTION_EXPRESSION = "transactionId"; + public static final Option<String> COMPRESS_CODEC = Options.key("compress_codec") + .stringType() + .noDefaultValue() + .withDescription("Compression codec"); + public static final Option<String> DATE_FORMAT = Options.key("date_format") + .stringType() + .defaultValue(DateUtils.Formatter.YYYY_MM_DD.getValue()) + .withDescription("Date format"); + public static final Option<String> DATETIME_FORMAT = Options.key("datetime_format") Review Comment: same as above. ########## seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseSourceConfig.java: ########## @@ -17,13 +17,39 @@ package org.apache.seatunnel.connectors.seatunnel.file.config; +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.common.utils.DateTimeUtils; +import org.apache.seatunnel.common.utils.DateUtils; +import org.apache.seatunnel.common.utils.TimeUtils; + public class BaseSourceConfig { - public static final String FILE_TYPE = "type"; - public static final String FILE_PATH = "path"; - public static final String SCHEMA = "schema"; - public static final String DELIMITER = "delimiter"; - public static final String DATE_FORMAT = "date_format"; - public static final String DATETIME_FORMAT = "datetime_format"; - public static final String TIME_FORMAT = "time_format"; - public static final String PARSE_PARTITION_FROM_PATH = "parse_partition_from_path"; + public static final Option<String> FILE_TYPE = Options.key("type") + .stringType() + .noDefaultValue() + .withDescription("File type"); + public static final Option<String> FILE_PATH = Options.key("path") + .stringType() + .noDefaultValue() + .withDescription("File path"); + public static final Option<String> DELIMITER = Options.key("delimiter") + .stringType() + .defaultValue(String.valueOf('\001')) + .withDescription("Filed delimiter"); + public static final Option<String> DATE_FORMAT = Options.key("date_format") + .stringType() + .defaultValue(DateUtils.Formatter.YYYY_MM_DD.getValue()) + .withDescription("Date format"); + public static final Option<String> DATETIME_FORMAT = Options.key("datetime_format") + .stringType() + .defaultValue(DateTimeUtils.Formatter.YYYY_MM_DD_HH_MM_SS.getValue()) + .withDescription("Datetime format"); + public static final Option<String> TIME_FORMAT = Options.key("time_format") + .stringType() + .defaultValue(TimeUtils.Formatter.HH_MM_SS.getValue()) + .withDescription("Time format"); + public static final Option<Boolean> PARSE_PARTITION_FROM_PATH = Options.key("parse_partition_from_path") + .booleanType() + .defaultValue(true) + .withDescription("Is or not parse partition fields from file path"); Review Comment: ```suggestion .withDescription("Wether parse partition fields from file path"); ``` ########## seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseSinkConfig.java: ########## @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.file.config; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.common.utils.DateTimeUtils; +import org.apache.seatunnel.common.utils.DateUtils; +import org.apache.seatunnel.common.utils.TimeUtils; + +import java.util.List; + +public class BaseSinkConfig { + public static final String SEATUNNEL = "seatunnel"; + public static final String NON_PARTITION = "NON_PARTITION"; + public static final String TRANSACTION_ID_SPLIT = "_"; + public static final String TRANSACTION_EXPRESSION = "transactionId"; + public static final Option<String> COMPRESS_CODEC = Options.key("compress_codec") + .stringType() + .noDefaultValue() + .withDescription("Compression codec"); + public static final Option<String> DATE_FORMAT = Options.key("date_format") + .stringType() + .defaultValue(DateUtils.Formatter.YYYY_MM_DD.getValue()) + .withDescription("Date format"); + public static final Option<String> DATETIME_FORMAT = Options.key("datetime_format") + .stringType() + .defaultValue(DateTimeUtils.Formatter.YYYY_MM_DD_HH_MM_SS.getValue()) + .withDescription("Datetime format"); + public static final Option<String> TIME_FORMAT = Options.key("time_format") + .stringType() + .defaultValue(TimeUtils.Formatter.HH_MM_SS.getValue()) + .withDescription("Time format"); + public static final Option<String> FILE_PATH = Options.key("path") + .stringType() + .noDefaultValue() + .withDescription("File path"); + public static final Option<String> FIELD_DELIMITER = Options.key("field_delimiter") + .stringType() + .defaultValue(String.valueOf(String.valueOf('\001'))) + .withDescription("Field delimiter"); + public static final Option<String> ROW_DELIMITER = Options.key("row_delimiter") + .stringType() + .defaultValue("\n") + .withDescription("Row delimiter"); + public static final Option<List<String>> PARTITION_BY = Options.key("partition_by") + .listType() + .noDefaultValue() + .withDescription("Partition keys list"); + public static final Option<String> PARTITION_DIR_EXPRESSION = Options.key("partition_dir_expression") + .stringType() + .defaultValue("${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/") + .withDescription("Partition directory expressions"); + public static final Option<Boolean> IS_PARTITION_FIELD_WRITE_IN_FILE = Options.key("is_partition_field_write_in_file") + .booleanType() + .defaultValue(false) + .withDescription("Whether to write partition fields to file"); + public static final Option<String> TMP_PATH = Options.key("tmp_path") + .stringType() + .defaultValue("/tmp/seatunnel") + .withDescription("Data write temporary path"); + public static final Option<String> FILE_NAME_EXPRESSION = Options.key("file_name_expression") + .stringType() + .defaultValue("${transactionId}") + .withDescription("Describe the file expression which will be created into the path"); Review Comment: ```suggestion .withDescription("`file_name_expression` describes the file expression which will be created into the `path`. We can add the variable `${now}` or `${uuid}` in the `file_name_expression`, like `test_${uuid}_${now}`,`${now}` represents the current time, and its format can be defined by specifying the option `filename_time_format`."); ``` ########## seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseSinkConfig.java: ########## @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.file.config; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.common.utils.DateTimeUtils; +import org.apache.seatunnel.common.utils.DateUtils; +import org.apache.seatunnel.common.utils.TimeUtils; + +import java.util.List; + +public class BaseSinkConfig { + public static final String SEATUNNEL = "seatunnel"; + public static final String NON_PARTITION = "NON_PARTITION"; + public static final String TRANSACTION_ID_SPLIT = "_"; + public static final String TRANSACTION_EXPRESSION = "transactionId"; + public static final Option<String> COMPRESS_CODEC = Options.key("compress_codec") + .stringType() + .noDefaultValue() + .withDescription("Compression codec"); + public static final Option<String> DATE_FORMAT = Options.key("date_format") + .stringType() + .defaultValue(DateUtils.Formatter.YYYY_MM_DD.getValue()) + .withDescription("Date format"); + public static final Option<String> DATETIME_FORMAT = Options.key("datetime_format") + .stringType() + .defaultValue(DateTimeUtils.Formatter.YYYY_MM_DD_HH_MM_SS.getValue()) + .withDescription("Datetime format"); + public static final Option<String> TIME_FORMAT = Options.key("time_format") + .stringType() + .defaultValue(TimeUtils.Formatter.HH_MM_SS.getValue()) + .withDescription("Time format"); + public static final Option<String> FILE_PATH = Options.key("path") + .stringType() + .noDefaultValue() + .withDescription("File path"); Review Comment: ```suggestion .withDescription("The target dir path"); ``` ########## seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseSinkConfig.java: ########## @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.file.config; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.common.utils.DateTimeUtils; +import org.apache.seatunnel.common.utils.DateUtils; +import org.apache.seatunnel.common.utils.TimeUtils; + +import java.util.List; + +public class BaseSinkConfig { + public static final String SEATUNNEL = "seatunnel"; + public static final String NON_PARTITION = "NON_PARTITION"; + public static final String TRANSACTION_ID_SPLIT = "_"; + public static final String TRANSACTION_EXPRESSION = "transactionId"; + public static final Option<String> COMPRESS_CODEC = Options.key("compress_codec") + .stringType() + .noDefaultValue() + .withDescription("Compression codec"); + public static final Option<String> DATE_FORMAT = Options.key("date_format") + .stringType() + .defaultValue(DateUtils.Formatter.YYYY_MM_DD.getValue()) + .withDescription("Date format"); + public static final Option<String> DATETIME_FORMAT = Options.key("datetime_format") + .stringType() + .defaultValue(DateTimeUtils.Formatter.YYYY_MM_DD_HH_MM_SS.getValue()) + .withDescription("Datetime format"); + public static final Option<String> TIME_FORMAT = Options.key("time_format") + .stringType() + .defaultValue(TimeUtils.Formatter.HH_MM_SS.getValue()) + .withDescription("Time format"); + public static final Option<String> FILE_PATH = Options.key("path") + .stringType() + .noDefaultValue() + .withDescription("File path"); + public static final Option<String> FIELD_DELIMITER = Options.key("field_delimiter") + .stringType() + .defaultValue(String.valueOf(String.valueOf('\001'))) + .withDescription("Field delimiter"); Review Comment: ```suggestion .withDescription("The separator between columns in a row of data. Only needed by `text` and `csv` file format."); ``` ########## seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseSinkConfig.java: ########## @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.file.config; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.common.utils.DateTimeUtils; +import org.apache.seatunnel.common.utils.DateUtils; +import org.apache.seatunnel.common.utils.TimeUtils; + +import java.util.List; + +public class BaseSinkConfig { + public static final String SEATUNNEL = "seatunnel"; + public static final String NON_PARTITION = "NON_PARTITION"; + public static final String TRANSACTION_ID_SPLIT = "_"; + public static final String TRANSACTION_EXPRESSION = "transactionId"; + public static final Option<String> COMPRESS_CODEC = Options.key("compress_codec") + .stringType() + .noDefaultValue() + .withDescription("Compression codec"); + public static final Option<String> DATE_FORMAT = Options.key("date_format") + .stringType() + .defaultValue(DateUtils.Formatter.YYYY_MM_DD.getValue()) + .withDescription("Date format"); + public static final Option<String> DATETIME_FORMAT = Options.key("datetime_format") + .stringType() + .defaultValue(DateTimeUtils.Formatter.YYYY_MM_DD_HH_MM_SS.getValue()) + .withDescription("Datetime format"); + public static final Option<String> TIME_FORMAT = Options.key("time_format") + .stringType() + .defaultValue(TimeUtils.Formatter.HH_MM_SS.getValue()) + .withDescription("Time format"); + public static final Option<String> FILE_PATH = Options.key("path") + .stringType() + .noDefaultValue() + .withDescription("File path"); + public static final Option<String> FIELD_DELIMITER = Options.key("field_delimiter") + .stringType() + .defaultValue(String.valueOf(String.valueOf('\001'))) + .withDescription("Field delimiter"); + public static final Option<String> ROW_DELIMITER = Options.key("row_delimiter") + .stringType() + .defaultValue("\n") + .withDescription("Row delimiter"); Review Comment: ```suggestion .withDescription("The separator between rows in a file. Only needed by `text` and `csv` file format."); ``` ########## seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseSinkConfig.java: ########## @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.file.config; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.common.utils.DateTimeUtils; +import org.apache.seatunnel.common.utils.DateUtils; +import org.apache.seatunnel.common.utils.TimeUtils; + +import java.util.List; + +public class BaseSinkConfig { + public static final String SEATUNNEL = "seatunnel"; + public static final String NON_PARTITION = "NON_PARTITION"; + public static final String TRANSACTION_ID_SPLIT = "_"; + public static final String TRANSACTION_EXPRESSION = "transactionId"; + public static final Option<String> COMPRESS_CODEC = Options.key("compress_codec") + .stringType() + .noDefaultValue() + .withDescription("Compression codec"); + public static final Option<String> DATE_FORMAT = Options.key("date_format") + .stringType() + .defaultValue(DateUtils.Formatter.YYYY_MM_DD.getValue()) + .withDescription("Date format"); + public static final Option<String> DATETIME_FORMAT = Options.key("datetime_format") + .stringType() + .defaultValue(DateTimeUtils.Formatter.YYYY_MM_DD_HH_MM_SS.getValue()) + .withDescription("Datetime format"); + public static final Option<String> TIME_FORMAT = Options.key("time_format") + .stringType() + .defaultValue(TimeUtils.Formatter.HH_MM_SS.getValue()) + .withDescription("Time format"); + public static final Option<String> FILE_PATH = Options.key("path") + .stringType() + .noDefaultValue() + .withDescription("File path"); + public static final Option<String> FIELD_DELIMITER = Options.key("field_delimiter") + .stringType() + .defaultValue(String.valueOf(String.valueOf('\001'))) + .withDescription("Field delimiter"); + public static final Option<String> ROW_DELIMITER = Options.key("row_delimiter") + .stringType() + .defaultValue("\n") + .withDescription("Row delimiter"); + public static final Option<List<String>> PARTITION_BY = Options.key("partition_by") + .listType() + .noDefaultValue() + .withDescription("Partition keys list"); + public static final Option<String> PARTITION_DIR_EXPRESSION = Options.key("partition_dir_expression") + .stringType() + .defaultValue("${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/") + .withDescription("Partition directory expressions"); Review Comment: ```suggestion .withDescription("If the `partition_by` is specified, we will generate the corresponding partition directory based on the partition information, and the final file will be placed in the partition directory. Default `partition_dir_expression` is `${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/`. `k0` is the first partition field and `v0` is the value of the first partition field."); ``` ########## seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseSinkConfig.java: ########## @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.file.config; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.common.utils.DateTimeUtils; +import org.apache.seatunnel.common.utils.DateUtils; +import org.apache.seatunnel.common.utils.TimeUtils; + +import java.util.List; + +public class BaseSinkConfig { + public static final String SEATUNNEL = "seatunnel"; + public static final String NON_PARTITION = "NON_PARTITION"; + public static final String TRANSACTION_ID_SPLIT = "_"; + public static final String TRANSACTION_EXPRESSION = "transactionId"; + public static final Option<String> COMPRESS_CODEC = Options.key("compress_codec") + .stringType() + .noDefaultValue() + .withDescription("Compression codec"); + public static final Option<String> DATE_FORMAT = Options.key("date_format") + .stringType() + .defaultValue(DateUtils.Formatter.YYYY_MM_DD.getValue()) + .withDescription("Date format"); + public static final Option<String> DATETIME_FORMAT = Options.key("datetime_format") + .stringType() + .defaultValue(DateTimeUtils.Formatter.YYYY_MM_DD_HH_MM_SS.getValue()) + .withDescription("Datetime format"); + public static final Option<String> TIME_FORMAT = Options.key("time_format") Review Comment: same as above. ########## seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseSourceConfig.java: ########## @@ -17,13 +17,39 @@ package org.apache.seatunnel.connectors.seatunnel.file.config; +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.common.utils.DateTimeUtils; +import org.apache.seatunnel.common.utils.DateUtils; +import org.apache.seatunnel.common.utils.TimeUtils; + public class BaseSourceConfig { - public static final String FILE_TYPE = "type"; - public static final String FILE_PATH = "path"; - public static final String SCHEMA = "schema"; - public static final String DELIMITER = "delimiter"; - public static final String DATE_FORMAT = "date_format"; - public static final String DATETIME_FORMAT = "datetime_format"; - public static final String TIME_FORMAT = "time_format"; - public static final String PARSE_PARTITION_FROM_PATH = "parse_partition_from_path"; + public static final Option<String> FILE_TYPE = Options.key("type") Review Comment: Use Enum better. We need let web ui know the value list. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
