pruidong opened a new issue, #9123:
URL: https://github.com/apache/seatunnel/issues/9123
Oracle migrates data to Clickhouse. Some fields in Oracle tables are null
values, and some date fields have the value of 1900-01-01 (ClickHouse only
supports 1970-01-01). Every column in the ClickHouse table is non-null, and
about 40 tables need to be synchronized. Is there any way to solve this
quickly? Thank you.
I tested it once using the following method, but there was no effect:
```
env {
execution.parallelism = 64
log_level = "DEBUG"
}
source {
Jdbc {
plugin_output = "ora_114"
driver = "oracle.jdbc.OracleDriver"
url = "jdbc:oracle:thin:@192.168.6.114:1521:ORCL"
user = "test"
password = "123456"
query = "SELECT * FROM TJ.${table} WHERE ${date_column} BETWEEN
TO_DATE('${start_date}','YYYY-MM-DD') AND TO_DATE('${end_date}','YYYY-MM-DD')"
fetch_size = 20000
}
}
transform {
DynamicCompile {
plugin_input = ["ora_114"]
plugin_output = "setting_default_value"
compile_language="JAVA"
compile_pattern="SOURCE_CODE"
source_code="""
import org.apache.seatunnel.api.table.catalog.Column;
import
org.apache.seatunnel.api.table.type.SeaTunnelRowAccessor;
import org.apache.seatunnel.api.table.catalog.*;
import org.apache.seatunnel.api.table.type.*;
import java.util.ArrayList;
import cn.hutool.json.JSONUtil;
public Column[] getInlineOutputColumns(CatalogTable
inputCatalogTable) {
return new Column[]{};
}
public Object[]
getInlineOutputFieldValues(SeaTunnelRowAccessor inputRow) {
Object[] fieldValues = new Object[2];
System.out.println("-------------------------------------------------------------------------------");
Object ageField = inputRow.getField(1);
System.out.println("-------------------------------------------------------------------------------");
System.out.println(JSONUtil.toJsonStr(inputRow));
System.out.println("-------------------------------------------------------------------------------");
fieldValues[0] = "JAVA";
return fieldValues;
}
"""
}
}
sink {
Clickhouse {
plugin_input = ["setting_default_value"]
host = "192.168.6.111:8123"
database = "TJ1"
table = "${table}"
username = "default"
password = "123456"
batch_size = 20000
fields = ["*"]
error_handle = "IGNORE"
retry {
max_retries = 5
interval_ms = 5000
}
error_log {
enabled = true
path = "../logs/seatunnel_clickhouse_errors.log"
}
skip_errors = ["Cannot set null to non-nullable column", "DateTime
should between"]
allow_nullable = true
}
}
common {
job.mode = "BATCH"
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
