This is an automated email from the ASF dual-hosted git repository.
diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-flink-connector.git
The following commit(s) were added to refs/heads/master by this push:
new f8743b80 [Fix] Fix the ora-12733 issue when there are many sync tables
(#532)
f8743b80 is described below
commit f8743b802cf016e4a8287aafa3c14dabc853d596
Author: wudi <[email protected]>
AuthorDate: Thu Jan 2 11:23:32 2025 +0800
[Fix] Fix the ora-12733 issue when there are many sync tables (#532)
---
.../org/apache/doris/flink/tools/cdc/DatabaseSync.java | 11 ++++++++++-
.../doris/flink/tools/cdc/oracle/OracleDatabaseSync.java | 15 +++++----------
2 files changed, 15 insertions(+), 11 deletions(-)
diff --git
a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java
b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java
index 701bd967..0c1b860a 100644
---
a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java
+++
b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java
@@ -42,6 +42,7 @@ import org.apache.doris.flink.sink.writer.WriteMode;
import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer;
import
org.apache.doris.flink.sink.writer.serializer.JsonDebeziumSchemaSerializer;
import org.apache.doris.flink.table.DorisConfigOptions;
+import org.apache.doris.flink.tools.cdc.oracle.OracleDatabaseSync;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -58,6 +59,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import static
org.apache.flink.cdc.debezium.utils.JdbcUrlUtils.PROPERTIES_PREFIX;
@@ -359,7 +361,14 @@ public abstract class DatabaseSync {
protected String getSyncTableList(List<String> syncTables) {
if (!singleSink) {
- return String.format("(%s)\\.(%s)", getTableListPrefix(),
String.join("|", syncTables));
+ if (this instanceof OracleDatabaseSync) {
+ return syncTables.stream()
+ .map(v -> getTableListPrefix() + "\\." + v)
+ .collect(Collectors.joining("|"));
+ } else {
+ return String.format(
+ "(%s)\\.(%s)", getTableListPrefix(), String.join("|",
syncTables));
+ }
} else {
// includingTablePattern and ^excludingPattern
if (includingTables == null) {
diff --git
a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java
b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java
index 1a6a4187..e529fe48 100644
---
a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java
+++
b/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java
@@ -164,16 +164,11 @@ public class OracleDatabaseSync extends DatabaseSync {
Preconditions.checkNotNull(databaseName, "database-name in oracle is
required");
Preconditions.checkNotNull(schemaName, "schema-name in oracle is
required");
String tableName = config.get(OracleSourceOptions.TABLE_NAME);
- // When debezium incrementally reads (refer
LogMinerQueryBuilder.listOfPatternsToSql),
- // it will be judged based on regexp_like. When the regular length
exceeds 512, an error
- // will be reported, like ORA-12733: regular expression too long
- if (tableName.length() > 450) {
- // REGEXP_LIKE('^SCHEMA.(TBL1|TBL2)$')
- if (StringUtils.isNullOrWhitespaceOnly(excludingTables)
- && (StringUtils.isNullOrWhitespaceOnly(includingTables)
- || ".*".equals(includingTables))) {
- tableName = ".*";
- }
+ // LogMinerQueryBuilder.buildTablePredicate is separated by commas to
avoid
+ // the error ORA-12733 when the regexp_like regular expression exceeds
512 characters
+ if (!singleSink && tableName.length() > 256) {
+ // todo: Make the length of a single regular expression as long as
possible.
+ tableName = tableName.replace("|", ",");
}
String url = config.get(OracleSourceOptions.URL);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
