This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch branch-2.0-var
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0-var by this push:
new ae02f873aab change version to 2.0.3-rc06-var06 (#28547)
ae02f873aab is described below
commit ae02f873aabc335f7410f3553d0d552c95bbdf6e
Author: lihangyu <[email protected]>
AuthorDate: Mon Dec 18 14:43:45 2023 +0800
change version to 2.0.3-rc06-var06 (#28547)
* change version to 2.0.3-rc05-var06
* [hotfix](jdbc catalog) fix jdbcmysqlclient npe (#28022)
* [Fix](schema change) disable convert light schema change (#28205)
* [Fix](inverted index) fix compound directory flush buffer error (#28191)
* [Fix](inverted index) fix need read data optimize problem (#28104)
---------
Co-authored-by: zy-kkk <[email protected]>
Co-authored-by: Luwei <[email protected]>
Co-authored-by: airborne12 <[email protected]>
---
.../segment_v2/inverted_index_compound_directory.cpp | 16 ++++++++++++++++
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 16 +++++++++++++---
.../src/main/java/org/apache/doris/common/Config.java | 2 +-
.../doris/datasource/jdbc/client/JdbcMySQLClient.java | 16 ++++++++++------
gensrc/script/gen_build_version.sh | 2 +-
.../test_index_compound_directory_fault_injection.out | 14 ++++++++++----
...est_index_compound_directory_fault_injection.groovy | 18 +++++++++++++++++-
7 files changed, 68 insertions(+), 16 deletions(-)
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp
index 4b9ff7d8dae..d7134b49116 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp
@@ -432,9 +432,25 @@ DorisCompoundDirectory::FSIndexOutput::~FSIndexOutput() {
void DorisCompoundDirectory::FSIndexOutput::flushBuffer(const uint8_t* b,
const int32_t size) {
if (_writer != nullptr && b != nullptr && size > 0) {
Slice data {b, (size_t)size};
+ DBUG_EXECUTE_IF(
+
"DorisCompoundDirectory::FSIndexOutput._mock_append_data_error_in_fsindexoutput_"
+ "flushBuffer",
+ {
+ if (_writer->path().filename() == "_0.tii" ||
+ _writer->path().filename() == "_0.tis") {
+ return;
+ }
+ })
Status st = _writer->append(data);
+ DBUG_EXECUTE_IF(
+
"DorisCompoundDirectory::FSIndexOutput._status_error_in_fsindexoutput_flushBuffer",
+ {
+ st =
Status::Error<doris::ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+ "flush buffer mock error");
+ })
if (!st.ok()) {
LOG(WARNING) << "File IO Write error: " << st.to_string();
+ _CLTHROWA(CL_ERR_IO, "writer append data when flushBuffer error");
}
} else {
if (_writer == nullptr) {
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 972f3f967e3..f2418cc6df2 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -850,6 +850,7 @@ Status
SegmentIterator::_apply_index_except_leafnode_of_andnode() {
if (_downgrade_without_index(res, need_remaining_after_evaluate)) {
// downgrade without index query
_not_apply_index_pred.insert(pred->column_id());
+ _need_read_data_indices[pred->column_id()] = true;
continue;
}
LOG(WARNING) << "failed to evaluate index"
@@ -869,7 +870,10 @@ Status
SegmentIterator::_apply_index_except_leafnode_of_andnode() {
_check_column_pred_all_push_down(column_name, true,
pred->type() ==
PredicateType::MATCH) &&
!pred->predicate_params()->marked_by_runtime_filter) {
- _need_read_data_indices[pred->column_id()] = false;
+ // if column's need_read_data already set true, we can not set it
to false now.
+ if (_need_read_data_indices.find(pred->column_id()) ==
_need_read_data_indices.end()) {
+ _need_read_data_indices[pred->column_id()] = false;
+ }
}
}
@@ -955,6 +959,7 @@ Status
SegmentIterator::_apply_inverted_index_on_column_predicate(
if (!res.ok()) {
if (_downgrade_without_index(res, need_remaining_after_evaluate)) {
remaining_predicates.emplace_back(pred);
+ _need_read_data_indices[pred->column_id()] = true;
return Status::OK();
}
LOG(WARNING) << "failed to evaluate index"
@@ -985,7 +990,10 @@ Status
SegmentIterator::_apply_inverted_index_on_column_predicate(
if (_check_column_pred_all_push_down(column_name, false,
pred->type() ==
PredicateType::MATCH) &&
!pred->predicate_params()->marked_by_runtime_filter) {
- _need_read_data_indices[pred->column_id()] = false;
+ // if column's need_read_data already set true, we can not set it
to false now.
+ if (_need_read_data_indices.find(pred->column_id()) ==
_need_read_data_indices.end()) {
+ _need_read_data_indices[pred->column_id()] = false;
+ }
}
}
return Status::OK();
@@ -1017,7 +1025,9 @@ Status
SegmentIterator::_apply_inverted_index_on_block_column_predicate(
if (res.ok()) {
if (_check_column_pred_all_push_down(column_name) &&
!all_predicates_are_marked_by_runtime_filter(predicate_set)) {
- _need_read_data_indices[column_id] = false;
+ if (_need_read_data_indices.find(column_id) ==
_need_read_data_indices.end()) {
+ _need_read_data_indices[column_id] = false;
+ }
}
no_need_to_pass_column_predicate_set.insert(predicate_set.begin(),
predicate_set.end());
_row_bitmap &= output_result;
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 0594c808307..907d02ffc26 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2122,7 +2122,7 @@ public class Config extends ConfigBase {
"暂时性配置项，开启后会自动将所有的olap表修改为可light schema change",
"temporary config filed, will make all olap tables enable light
schema change"
})
- public static boolean enable_convert_light_weight_schema_change = true;
+ public static boolean enable_convert_light_weight_schema_change = false;
@ConfField(mutable = true, masterOnly = false, description = {
"查询information_schema.metadata_name_ids表时,获取一个数据库中所有表用的时间",
"When querying the information_schema.metadata_name_ids table,"
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
index 61ba2a0db47..c8506cc99a9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
@@ -119,15 +119,19 @@ public class JdbcMySQLClient extends JdbcClient {
List<JdbcFieldSchema> tableSchema =
com.google.common.collect.Lists.newArrayList();
// if isLowerCaseTableNames == true, tableName is lower case
// but databaseMetaData.getColumns() is case sensitive
+ String currentDbName = dbName;
+ String currentTableName = tableName;
if (isLowerCaseTableNames) {
- dbName = lowerDBToRealDB.get(dbName);
- tableName = lowerTableToRealTable.get(tableName);
+ currentDbName = getRealDatabaseName(dbName);
+ currentTableName = getRealTableName(dbName, tableName);
}
+ String finalDbName = currentDbName;
+ String finalTableName = currentTableName;
try {
DatabaseMetaData databaseMetaData = conn.getMetaData();
String catalogName = getCatalogName(conn);
- rs = getColumns(databaseMetaData, catalogName, dbName, tableName);
- List<String> primaryKeys = getPrimaryKeys(databaseMetaData,
catalogName, dbName, tableName);
+ rs = getColumns(databaseMetaData, catalogName, finalDbName,
finalTableName);
+ List<String> primaryKeys = getPrimaryKeys(databaseMetaData,
catalogName, finalDbName, finalTableName);
Map<String, String> mapFieldtoType = null;
while (rs.next()) {
JdbcFieldSchema field = new JdbcFieldSchema();
@@ -140,7 +144,7 @@ public class JdbcMySQLClient extends JdbcClient {
// in mysql-jdbc-connector-5.1.*, TYPE_NAME of BITMAP column
in doris will be "BITMAP"
field.setDataTypeName(rs.getString("TYPE_NAME"));
if (isDoris) {
- mapFieldtoType = getColumnsDataTypeUseQuery(dbName,
tableName);
+ mapFieldtoType = getColumnsDataTypeUseQuery(finalDbName,
finalTableName);
field.setDataTypeName(mapFieldtoType.get(rs.getString("COLUMN_NAME")));
}
field.setKey(primaryKeys.contains(field.getColumnName()));
@@ -162,7 +166,7 @@ public class JdbcMySQLClient extends JdbcClient {
tableSchema.add(field);
}
} catch (SQLException e) {
- throw new JdbcClientException("failed to get table name list from
jdbc for table %s:%s", e, tableName,
+ throw new JdbcClientException("failed to get table name list from
jdbc for table %s:%s", e, finalTableName,
Util.getRootCauseMessage(e));
} finally {
close(rs, conn);
diff --git a/gensrc/script/gen_build_version.sh
b/gensrc/script/gen_build_version.sh
index e850b33bca5..91e404246be 100755
--- a/gensrc/script/gen_build_version.sh
+++ b/gensrc/script/gen_build_version.sh
@@ -31,7 +31,7 @@ build_version_prefix="doris"
build_version_major=2
build_version_minor=0
build_version_patch=3
-build_version_rc_version="rc05-var05"
+build_version_rc_version="rc05-var06"
build_version="${build_version_prefix}-${build_version_major}.${build_version_minor}.${build_version_patch}-${build_version_rc_version}"
diff --git
a/regression-test/data/fault_injection_p0/test_index_compound_directory_fault_injection.out
b/regression-test/data/fault_injection_p0/test_index_compound_directory_fault_injection.out
index 89a6bba7134..bf59811cc65 100644
---
a/regression-test/data/fault_injection_p0/test_index_compound_directory_fault_injection.out
+++
b/regression-test/data/fault_injection_p0/test_index_compound_directory_fault_injection.out
@@ -1,13 +1,19 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !sql --
-863
+0
-- !sql --
-863
+0
-- !sql --
-863
+0
-- !sql --
-863
+0
+
+-- !sql --
+0
+
+-- !sql --
+0
diff --git
a/regression-test/suites/fault_injection_p0/test_index_compound_directory_fault_injection.groovy
b/regression-test/suites/fault_injection_p0/test_index_compound_directory_fault_injection.groovy
index e5d028a62ef..1c6f969dd2f 100644
---
a/regression-test/suites/fault_injection_p0/test_index_compound_directory_fault_injection.groovy
+++
b/regression-test/suites/fault_injection_p0/test_index_compound_directory_fault_injection.groovy
@@ -116,7 +116,23 @@ suite("test_index_compound_directory_failure_injection",
"nonConcurrent") {
} finally {
GetDebugPoint().disableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._set_writer_close_status_error")
}
- qt_sql "select COUNT() from ${testTable_dup} where request match
'images'"
+ qt_sql "select COUNT() from ${testTable_dup} where request match
'images'"
+ try {
+ create_httplogs_dup_table.call("test_index_compound_directory1")
+
GetDebugPoint().enableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._mock_append_data_error_in_fsindexoutput_flushBuffer")
+ load_httplogs_data.call("test_index_compound_directory1",
'test_index_compound_directory1', 'true', 'json', 'documents-1000.json')
+ } finally {
+
GetDebugPoint().disableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._mock_append_data_error_in_fsindexoutput_flushBuffer")
+ }
+ qt_sql "select COUNT() from test_index_compound_directory1 where
request match 'images'"
+ try {
+ create_httplogs_dup_table.call("test_index_compound_directory2")
+
GetDebugPoint().enableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._status_error_in_fsindexoutput_flushBuffer")
+ load_httplogs_data.call("test_index_compound_directory2",
'test_index_compound_directory2', 'true', 'json', 'documents-1000.json')
+ } finally {
+
GetDebugPoint().disableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._status_error_in_fsindexoutput_flushBuffer")
+ }
+ qt_sql "select COUNT() from test_index_compound_directory2 where
request match 'images'"
} finally {
//try_sql("DROP TABLE IF EXISTS ${testTable}")
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
