This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch master-catalog-spi-review-21 in repository https://gitbox.apache.org/repos/asf/doris.git
commit b97bf008ebcb19112617e658d749e1ea9b4438a9 Author: morningman <[email protected]> AuthorDate: Tue Jul 28 20:14:27 2026 +0800 [refactor](catalog) fe-core: remove the dead storage doors on the catalog base Deletes five callerless members from ExternalCatalog -- getHadoopProperties, getConfiguration, buildConf, buildHadoopConfiguration and ifNotSetFallbackToSimpleAuth -- along with the cachedConf/confLock fields they cached through, and three from CatalogProperty: getHadoopProperties, getBackendStorageProperties and getOrderedStorageAdapters, with their two lazily-populated fields. getConfiguration was already marked deprecated "until connector SPI extraction is complete", which it now is. Re-grepping before the cut corrected the plan twice. It had said not to touch buildHadoopConfiguration because its callers were never enumerated; enumerating them shows all sixteen hits are the connector-side IcebergCatalogFactory and PaimonCatalogFactory methods of the same name, in different classes, so the ExternalCatalog one is dead too. The plan had also missed ifNotSetFallbackToSimpleAuth, whose only two uses were inside the methods being deleted, and missed that cachedConf/confLock are also touched outside getConfiguration -- in resetToUninitialized and in the Gson post-process -- which a field-declaration-only deletion would have broken. The point is not the 135 lines. initStorageAdapters is now reachable only through getStorageAdaptersMap and getEffectiveRawStorageProperties, both called from PluginDrivenExternalCatalog, so "fe-core storage has a single entrance" holds by construction rather than by audit -- which is also what backs the fail-loud branch added when the metastore cluster was retired. Verified: repo-wide grep clean; full reactor clean test-compile including test sources; fe-core checkstyle 0 violations. The full fe-core suite was stopped by the owner at 3h29m after 1232 test classes rather than run to completion, so the test claim is "no failures among the 1232 classes executed", not "the suite passes". Its one failure, ForwardToMasterTest.testAddBeDropBe, reproduces identically on a clean HEAD with these changes stashed, so it is pre-existing. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> --- .../apache/doris/datasource/CatalogProperty.java | 65 -------------------- .../apache/doris/datasource/ExternalCatalog.java | 70 ---------------------- plan-doc/fecore-property-cleanup/HANDOFF.md | 69 +++++++++++---------- plan-doc/fecore-property-cleanup/progress.md | 66 ++++++++++++++++++++ plan-doc/fecore-property-cleanup/tasklist.md | 61 +++++++++++++++---- 5 files changed, 151 insertions(+), 180 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java index 2ad75d6b950..2fb365e9e4d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java @@ -23,7 +23,6 @@ import org.apache.doris.datasource.storage.StorageTypeId; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; import org.apache.commons.collections4.MapUtils; -import org.apache.hadoop.conf.Configuration; import java.util.Collections; import java.util.HashMap; @@ -82,12 +81,6 @@ public class CatalogProperty { } } - // Lazy-loaded backend storage properties, using volatile to ensure visibility - private volatile Map<String, String> backendStorageProperties; - - // Lazy-loaded Hadoop properties, using volatile to ensure visibility - private volatile Map<String, String> hadoopProperties; - // Design S8: for a plugin catalog, the connector owns storage-property derivation (e.g. iceberg hadoop // warehouse -> fs.defaultFS); this supplier returns the connector-derived storage defaults fe-core folds // into the storage map, so fe-core does NOT parse metastore properties on the storage path. Set once by @@ -176,8 +169,6 @@ public class CatalogProperty { */ private void resetAllCaches() { this.storageBindings = null; - this.backendStorageProperties = null; - this.hadoopProperties = null; } /** @@ -220,10 +211,6 @@ public class CatalogProperty { return initStorageAdapters().byType; } - public List<StorageAdapter> getOrderedStorageAdapters() { - return initStorageAdapters().ordered; - } - /** * The catalog's persisted user props merged with derived storage defaults, which the connector supplies * (design S8: {@link #pluginDerivedStorageDefaultsSupplier} — fe-core does not parse metastore properties @@ -295,56 +282,4 @@ public class CatalogProperty { } } - /** - * Get backend storage properties with lazy loading, using double-check locking to ensure thread safety - */ - public Map<String, String> getBackendStorageProperties() { - if (backendStorageProperties == null) { - synchronized (this) { - if (backendStorageProperties == null) { - Map<String, String> result = new HashMap<>(); - Map<StorageTypeId, StorageAdapter> storageMap = getStorageAdaptersMap(); - - for (StorageAdapter sp : storageMap.values()) { - Map<String, String> backendProps = sp.getBackendConfigProperties(); - // the backend property's value can not be null, because it will be serialized to thrift, - // which does not support null value. - backendProps.entrySet().stream().filter(e -> e.getValue() != null) - .forEach(e -> result.put(e.getKey(), e.getValue())); - } - - this.backendStorageProperties = result; - } - } - } - return backendStorageProperties; - } - - /** - * Get Hadoop properties with lazy loading, using double-check locking to ensure thread safety - */ - public Map<String, String> getHadoopProperties() { - if (hadoopProperties == null) { - synchronized (this) { - if (hadoopProperties == null) { - hadoopProperties = new HashMap<>(); - Map<StorageTypeId, StorageAdapter> storageMap = getStorageAdaptersMap(); - - for (StorageAdapter sp : storageMap.values()) { - Configuration configuration = sp.getHadoopStorageConfig(); - if (configuration != null) { - configuration.forEach(entry -> { - String key = entry.getKey(); - String value = entry.getValue(); - if (value != null) { - hadoopProperties.put(key, value); - } - }); - } - } - } - } - } - return hadoopProperties; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 32990344a9f..5fd8a88c3d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -64,8 +64,6 @@ import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.math.NumberUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; @@ -174,9 +172,6 @@ public abstract class ExternalCatalog // Map lowercase database names to actual remote database names for case-insensitive lookup private Map<String, String> lowerCaseToDatabaseName = Maps.newConcurrentMap(); - private volatile Configuration cachedConf = null; - private byte[] confLock = new byte[0]; - private volatile boolean isInitializing = false; public ExternalCatalog() { @@ -200,62 +195,6 @@ public abstract class ExternalCatalog } } - /** - * Returns Hadoop-related properties as a plain Map. - * Connector plugins should use this instead of getConfiguration() - * and build their own Configuration internally when needed. - */ - public Map<String, String> getHadoopProperties() { - Map<String, String> props = new java.util.HashMap<>(catalogProperty.getHadoopProperties()); - if (ifNotSetFallbackToSimpleAuth()) { - props.putIfAbsent("ipc.client.fallback-to-simple-auth-allowed", "true"); - } - return props; - } - - /** - * @deprecated Use {@link #getHadoopProperties()} and build Configuration locally. - * This method will be removed when connector SPI extraction is complete. - */ - @Deprecated - public Configuration getConfiguration() { - // build configuration is costly, so we cache it. - if (cachedConf != null) { - return cachedConf; - } - synchronized (confLock) { - if (cachedConf != null) { - return cachedConf; - } - cachedConf = buildConf(); - return cachedConf; - } - } - - /** - * Builds a Hadoop Configuration from a properties map. - * Use this when you need a Configuration object from catalog properties. - */ - public static Configuration buildHadoopConfiguration(Map<String, String> properties) { - Configuration conf = new HdfsConfiguration(); - for (Map.Entry<String, String> entry : properties.entrySet()) { - conf.set(entry.getKey(), entry.getValue()); - } - return conf; - } - - private Configuration buildConf() { - Configuration conf = new HdfsConfiguration(); - if (ifNotSetFallbackToSimpleAuth()) { - conf.set("ipc.client.fallback-to-simple-auth-allowed", "true"); - } - Map<String, String> catalogProperties = catalogProperty.getHadoopProperties(); - for (Map.Entry<String, String> entry : catalogProperties.entrySet()) { - conf.set(entry.getKey(), entry.getValue()); - } - return conf; - } - /** * Lists all database names in this catalog. * @@ -287,11 +226,6 @@ public abstract class ExternalCatalog return catalogProperty.getEnableMappingTimestampTz(); } - // we need check auth fallback for kerberos or simple - public boolean ifNotSetFallbackToSimpleAuth() { - return catalogProperty.getOrDefault("ipc.client.fallback-to-simple-auth-allowed", "").isEmpty(); - } - // Will be called when creating catalog(not replaying). // Subclass can override this method to do some check when creating catalog. // Connectivity testing (test_connection=true) is connector-specific and therefore lives behind the @@ -675,9 +609,6 @@ public abstract class ExternalCatalog synchronized (this) { this.objectCreated = false; this.initialized = false; - synchronized (this.confLock) { - this.cachedConf = null; - } this.lowerCaseToDatabaseName.clear(); onClose(); } @@ -1109,7 +1040,6 @@ public abstract class ExternalCatalog } } } - this.confLock = new byte[0]; this.initialized = false; setDefaultPropsIfMissing(true); if (tableAutoAnalyzePolicy == null) { diff --git a/plan-doc/fecore-property-cleanup/HANDOFF.md b/plan-doc/fecore-property-cleanup/HANDOFF.md index dc8c881ace7..14c3b4367fa 100644 --- a/plan-doc/fecore-property-cleanup/HANDOFF.md +++ b/plan-doc/fecore-property-cleanup/HANDOFF.md @@ -7,7 +7,7 @@ --- -# ✅ 本任务核心工作已全部完成。只剩可选的 FPC-04。 +# ✅ 本任务空间核心工作（FPC-01 ~ FPC-04）已全部完成 **没有待拍板事项**（OD-1 / OD-2 均已由用户 2026-07-28 拍板并执行完毕）。 @@ -15,46 +15,45 @@ |---|---|---| | 文档空间 | `938d38c7425` | 6 份文档 | | FPC-01 + FPC-03 主删除 | `ac2d931ee3a` | 删 5 文件 473 行 + `CatalogProperty` 净减 ~45 行 | -| OD-2 反向发现 | `6d245a524d3` | 纯文档，无代码 | -| FPC-02 删死构造臂 | 见 `git log` | 实删 159 行 | +| OD-2 反向发现 | `6d245a524d3` | 纯文档 | +| FPC-02 删死构造臂 | `a824cd81ac1` | 实删 159 行 | +| FPC-04 清扫死 storage 门 | 见 `git log` | 纯删除 135 行，零新增 | -**fe-core `datasource/property/` 现状**：`metastore/` 已不存在；只剩 -`common`（已瘦身到只有活代码）/ `constants` / `fileformat`。 +`fe/fe-core/.../datasource/property/` 现在只剩 `common`（已瘦身到只有活代码）/ `constants` / `fileformat`； +`initStorageAdapters()` 的入口收敛为 `PluginDrivenExternalCatalog:207-208` 两处，**由构造保证**。 --- -## 📌 两条拍板结论（**已执行，勿再动摇**） +## ⚠️ FPC-04 的验证有一处缺口，接手时请知悉 -- **OD-1 = 抛异常。** `CatalogProperty.resolveDerivedStorageDefaults()` 的 null-supplier 分支 - `throw new IllegalStateException(...)`，守卫测试 - `CatalogPropertyPluginStorageDerivationTest.unwiredSupplierFailsLoudInsteadOfDerivingNothing` - （已做变异验证）。 -- **OD-2 = 直接删**（用户推翻了我的推荐）。**已知并接受的代价**： - `upstream-apache/master` 上 `StorageAdapter.getAwsCredentialsProvider()` **是活的** - （两个调用者在本分支已随 `datasource/connectivity/` 包一起删掉了）。 - ⇒ **上游改动该区域时 rebase 会出 modify/delete 冲突，届时保留删除** - （对齐本仓库既有范式：master 给已删子系统打的修复，解法是保留删除 + 必要时移植到连接器）。 +完整 fe-core 套件**未跑完**：跑到 **3h29m / 1232 个测试类**时由用户指示**主动终止** +（套件耗时问题已单独立档 [`../fe-core-ut-runtime-problem.md`](../fe-core-ut-runtime-problem.md)）。 ---- +- 已绿：残留 grep = 0 · 全反应堆 `test-compile`（含测试源）· `checkstyle:check` 0 violations +- 终止时唯一失败：`http.ForwardToMasterTest.testAddBeDropBe` + （`ClassCastException: JSONObject → JSONArray`），**已用 stash 回干净 HEAD 复现 ⇒ 既有失败，与本改动无关** +- **口径**：「已执行的 1232 个类中除 1 个既有失败外无失败」，**不是**「全套件通过」 + +⇒ 若要补齐，重跑一次完整套件即可（注意耗时）。 -## ⏭ 剩下的唯一任务：FPC-04（可选） +--- -清扫 fe-core 已死的 storage 门：`ExternalCatalog.getHadoopProperties()` / -`getConfiguration()`（已标 `@Deprecated`）+ `buildConf()` 及缓存字段、 -`CatalogProperty.getBackendStorageProperties()` / `getOrderedStorageAdapters()`。 +## ⏭ 本任务空间已完结 -- **动手前必须重新逐符号 grep 确认零调用者**（别信这份文档的旧结论）。 -- **✋ 不要碰** `ExternalCatalog.buildHadoopConfiguration(Map)` —— 它的调用者从未枚举过。 -- 它动的是**每个 catalog 都继承的基类** ⇒ 窄 `-Dtest` 列表不够，要跑 - `mvn -pl fe-core -am test -Dcheckstyle.skip=true -DfailIfNoTests=false --fail-at-end`。 -- 收益：做完后 `PluginDrivenExternalCatalog:207-208` 成为 `initStorageAdapters()` 的 - **唯一入口（由构造保证，而非靠人工审计）**。 +剩下的都是**单列后续，不属于本空间**，见 `tasklist.md` 末尾 **SEP-1 ~ SEP-4**： +- **SEP-1** `StorageAdapter.checkAzureOauth2OnlyForIcebergRest()` 在存储路径读 metastore 命名空间键 + —— 真架构违规，但带上游 #66004 刻意的大小写敏感怪癖，需单独一刀 + e2e +- **SEP-2** 把 `S3CredentialsProviderType` 上提 `fe-filesystem-api` → 调和 `hadoopClassName` → 删 `common/` + —— 架构终局，但会改线上串，需用户先就 `Config.aws_credentials_provider_version` v1 分支拍板 +- **SEP-3** `BaseProperties.getCloudCredential()` 零调用者 +- **SEP-4** `fe-connector-metastore-api/pom.xml:64` 那句注释是过时的假话 -其余单列后续见 `tasklist.md` 末尾的 **SEP-1 ~ SEP-4**（都不属于本任务）。 +另有一条**独立线**：主线 `plan-doc/HANDOFF.md` 的 scope 是「修 TeamCity **CI 997422** +（Doris_External_Regression）失败用例」。**其当前红绿状态本 session 未查证。** --- -## ⚠️ 五条验证纪律（第 4、5 条是这两轮实测新增的） +## ⚠️ 五条验证纪律 1. 删除类改动**不能只信增量编译** → 每步先 `rm -rf fe-core/target/{classes,test-classes}`。 2. 全反应堆**必须含测试源**（禁 `-Dmaven.test.skip=true`），且必须 `-Dcheckstyle.skip=true`。 @@ -63,8 +62,11 @@ 且 surefire 2.22.2 认 **`-DfailIfNoTests=false`**（不是 `-DfailIfNoSpecifiedTests`）。 5. **但 `-am test` 对「依赖链经过 shade 模块」的连接器跑不通**（如 iceberg → hms： 报 `package org.apache.hadoop.hive.metastore.api does not exist`，因为 shaded jar 只在 - `package` 阶段产出）。**这是既有怪癖，已 stash 到干净 HEAD 复现确认。** - 这类模块用全反应堆 `test-compile` 覆盖；`fe-connector-api` 不在该链上，`-am test` 正常。 + `package` 阶段产出）。**既有怪癖，已 stash 到干净 HEAD 复现确认。** + 这类模块用全反应堆 `test-compile` 覆盖；`fe-core` / `fe-connector-api` 不在该链上，`-am test` 正常。 + +**删字段时额外注意**（FPC-04 实证）：先 grep 字段名的**全部**出现位置， +方法外的使用（cache reset / 反序列化后处理）最容易漏，只删声明会编译不过。 --- @@ -85,6 +87,9 @@ ## 🔎 尚未验证（如实声明） -- **没跑 e2e**（需要集群）。两次删除都是删不可达代码，且 Gson 回放 + 存储适配对齐测试已过， +- **完整 fe-core 套件未跑完**（见上方 ⚠️ 段），口径是「已执行的 1232 类中除 1 个既有失败外无失败」。 +- **没跑 e2e**（需要集群）。四次删除都是删不可达代码，Gson 回放 + 存储适配对齐测试已过， 但真正的存储绑定路径（iceberg hadoop `warehouse → fs.defaultFS`）只有单测覆盖。 -- `ExternalCatalog.buildHadoopConfiguration(Map)` 的调用者没枚举 ⇒ FPC-04 明确排除它。 +- **主线 CI 997422 的当前状态未查证。** +- OD-2 已知代价：`getAwsCredentialsProvider()` 在上游是活的 ⇒ 上游改动该区域时 rebase 会出 + modify/delete 冲突，**届时保留删除**。 diff --git a/plan-doc/fecore-property-cleanup/progress.md b/plan-doc/fecore-property-cleanup/progress.md index ecba6a404cd..a967b51b067 100644 --- a/plan-doc/fecore-property-cleanup/progress.md +++ b/plan-doc/fecore-property-cleanup/progress.md @@ -227,3 +227,69 @@ OD-2 被推翻是合理的：我的推荐建立在「为零功能收益承担 re `AwsCredentialsProviderFactory` 只剩 `getV2ClassName(mode, boolean)` + 两个 env 探针 （喂 `StorageAdapter:645/:654` 那条**活的** hadoop/BE 串）。 **本任务空间的核心工作（FPC-01/02/03）已全部完成**，只剩可选的 FPC-04。 + +--- + +## 2026-07-28（五）— FPC-04：清扫已死的 fe-core storage 门 + +**纯删除 135 行、零新增**（`ExternalCatalog` −70 / `CatalogProperty` −65）。 + +### 起手先重新侦察，结果修正了本空间文档两处 + +HANDOFF 写了「动手前必须重新逐符号 grep 确认零调用者（别信这份文档的旧结论）」——照做了， +**两处都被修正**： + +**修正 ①：`ExternalCatalog.buildHadoopConfiguration(Map)` 也是死的。** +文档原文是「✋ 不要碰它 —— 它的调用者从未枚举过」。枚举后发现：全仓 16 处 +`buildHadoopConfiguration` 命中**全部是连接器侧同名但不同类**的 +`IcebergCatalogFactory.buildHadoopConfiguration` / `PaimonCatalogFactory.buildHadoopConfiguration`， +`ExternalCatalog` 上那个 `public static` 方法**零调用**。 +⇒ 这是「谨慎导致的**低估**」，与本仓库常见的「蓝图高估工作量」正好相反，但根因同一条： +**别信文档信侦察**。 + +**修正 ②：文档漏了两个连带项。** +- `ExternalCatalog.ifNotSetFallbackToSimpleAuth()`（`public`）——全仓仅 2 处使用， + 且都在将死的 `getHadoopProperties()` / `buildConf()` 里 ⇒ 连带归零。 +- `cachedConf` / `confLock` 在**方法外**还有两处使用（`resetToUninitialized()` 里的置空块、 + 反序列化后处理里的 `this.confLock = new byte[0];`）。**只删字段不清这两处会编译不过**—— + 这类「字段的方法外使用」是删字段时最容易漏的一类，值得单独记一笔。 + +### 🎯 真正的收益不是行数 + +做完后 `initStorageAdapters()` 的入口只剩 `getStorageAdaptersMap()` 与 +`getEffectiveRawStorageProperties()`，且都来自 `PluginDrivenExternalCatalog:207-208` +⇒ **「fe-core 存储只有一个入口」从「靠人工审计」升级为「由构造保证」**， +正好给 FPC-03 落地的那个 fail-loud 兜底上了双保险： +既然入口唯一且都在 supplier 装好之后，那个 `throw` 就更不可能被误触发。 + +同时 `resetAllCaches()` 从 4 行瘦到 1 行（只剩 `storageBindings = null`）， +`CatalogProperty` 的可变缓存状态从 4 个字段收敛到 1 个。 + +### 验证 + +| 项 | 结果 | +|---|---| +| 全仓残留 grep（含 groovy） | **0** | +| 全反应堆 `clean test-compile -Dcheckstyle.skip=true`（含测试源） | **BUILD SUCCESS** | +| `-pl fe-core checkstyle:check` | **0 violations** | +| **完整 fe-core 单测**（`-am test --fail-at-end`） | ⏳ **待填** | + +⚠️ 最后一项**必须整套跑**：改的是每个 catalog 都继承的基类，窄 `-Dtest` 列表覆盖不到 +那些间接依赖基类行为的测试。按纪律，后台任务通知里的 exit code 是 `echo` 的不是 maven 的， +**以日志里的 `BUILD` 行和 `Tests run` 汇总行为准**。 + +### 验证结果（补记） + +**完整 fe-core 套件未跑完**：跑到 **3h29m / 1232 个测试类**时由用户指示主动终止 +（该套件本身的耗时问题已单独立档 `../fe-core-ut-runtime-problem.md`）。 + +终止时**只有 1 个测试类失败**：`http.ForwardToMasterTest.testAddBeDropBe`， +`ClassCastException: org.json.simple.JSONObject cannot be cast to JSONArray`。 + +**归因方式：stash 复现，不用推理。** `git stash push -u -- fe/` 回到干净 HEAD `2ecd7753766` +单跑该测试 → **一模一样地失败** ⇒ **既有失败，与 FPC-04 无关**。 +其余 1231 个测试类全绿，其中与本改动直接相关的 `datasource`(114) / `connector`(42) / +`filesystem`(70) / `persist`(37) 四片在头 20 分钟内就跑完且全绿。 + +⚠️ **如实声明**：该套件**未跑到 BUILD 汇总行**，所以「全绿」的口径是 +「已执行的 1232 个类中除 1 个既有失败外无失败」，**不是**「全套件通过」。 diff --git a/plan-doc/fecore-property-cleanup/tasklist.md b/plan-doc/fecore-property-cleanup/tasklist.md index 80142027987..c65ce0c01bf 100644 --- a/plan-doc/fecore-property-cleanup/tasklist.md +++ b/plan-doc/fecore-property-cleanup/tasklist.md @@ -167,19 +167,54 @@ mvn -f $R/fe/pom.xml -pl fe-core checkstyle:check --- -## 阶段 4 — 可选清扫（**另起提交，落地前重新 grep**） - -- [ ] **FPC-04** ⬜ 清扫 fe-core 已死的 storage 门 - - **仅当执行时重新 grep 确认零调用者**才做： - `ExternalCatalog.getHadoopProperties()`、`ExternalCatalog.getConfiguration()`（已标 `@Deprecated`） - + `buildConf()` 及其缓存字段、`CatalogProperty.getBackendStorageProperties()`、 - `CatalogProperty.getOrderedStorageAdapters()` - - **✋ 不要碰** `ExternalCatalog.buildHadoopConfiguration(Map)` —— 它的调用者**没有枚举过** - - **收益**：做完后 `PluginDrivenExternalCatalog.java:207-208` 成为 `initStorageAdapters()` 的 - **唯一入口（由构造保证，而非靠人工审计）** - - **验收**：逐符号零调用者 grep → **完整** `mvn -pl fe-core test -Dcheckstyle.skip=true --fail-at-end` - （⚠️ 它动的是每个 catalog 都继承的基类，**窄 `-Dtest` 列表不够**）→ `checkstyle:check` - - 🟢 刻意与 FPC-03 分开，好让 FPC-03 保持**可单独回滚** +## 阶段 4 — 清扫已死的 fe-core storage 门 + +- [x] **FPC-04** ✅ **已落地**（**纯删除 135 行，零新增**） + > 📌 **落地前重新 grep 的结果修正了本文档两处**（详见 `progress.md` 2026-07-28（五））： + > ① 原文写「✋ 不要碰 `ExternalCatalog.buildHadoopConfiguration(Map)` —— 其调用者未曾枚举」。 + > **枚举了：它也是死的。** 全仓 16 处 `buildHadoopConfiguration` 命中**全是连接器侧同名但不同类**的 + > `IcebergCatalogFactory.` / `PaimonCatalogFactory.` 方法，`ExternalCatalog` 上那个零调用。 + > ② 原文**漏了** `ExternalCatalog.ifNotSetFallbackToSimpleAuth()`（`public`，全仓仅 2 处使用 + > 且都在将死方法内 ⇒ 连带死亡），以及 `cachedConf`/`confLock` 在**方法外**还有两处使用。 + + - **`ExternalCatalog.java`（−70 行）** + - 删方法：`getHadoopProperties()` · `getConfiguration()`（`@Deprecated` 原注释就写着 + "will be removed when connector SPI extraction is complete"）· `buildConf()` · + `buildHadoopConfiguration(Map)` · `ifNotSetFallbackToSimpleAuth()` + - 删字段：`cachedConf` · `confLock` + - **两处方法外使用一并清掉**（易漏）：`resetToUninitialized()` 里的 + `synchronized (this.confLock) { this.cachedConf = null; }` 块；反序列化后处理里的 + `this.confLock = new byte[0];` + - 删孤儿 import：`Configuration` · `HdfsConfiguration` + - **`CatalogProperty.java`（−65 行）** + - 删方法：`getHadoopProperties()` · `getBackendStorageProperties()` · `getOrderedStorageAdapters()` + - 删字段：`hadoopProperties` · `backendStorageProperties` + ⇒ `resetAllCaches()` 瘦到只剩一行 `this.storageBindings = null;` + - 删孤儿 import：`Configuration` + - **🎯 真正的收益不在行数**：做完后 `initStorageAdapters()` 的入口只剩 + `getStorageAdaptersMap()` 与 `getEffectiveRawStorageProperties()`，且都来自 + `PluginDrivenExternalCatalog:207-208` ⇒ **「fe-core 存储只有一个入口」从「靠人工审计」 + 变成「由构造保证」**，正好给 FPC-03 那个 fail-loud 兜底上双保险。 + - **验收**： + ```bash + R=/mnt/disk1/yy/git/wt-catalog-spi + grep -rIn "ifNotSetFallbackToSimpleAuth\|getOrderedStorageAdapters\|\.buildHadoopConfiguration(\|catalogProperty\.getHadoopProperties\|catalogProperty\.getBackendStorageProperties" \ + $R/fe $R/regression-test --include=*.java --include=*.groovy --exclude-dir=target \ + | grep -v "IcebergCatalogFactory\.\|PaimonCatalogFactory\." # → 空 + rm -rf $R/fe/fe-core/target/classes $R/fe/fe-core/target/test-classes + mvn -f $R/fe/pom.xml -T 1C clean test-compile -Dcheckstyle.skip=true + mvn -f $R/fe/pom.xml -pl fe-core checkstyle:check + # 🔴 动的是每个 catalog 都继承的基类 ⇒ 窄 -Dtest 列表不够，必须整套 + --fail-at-end + mvn -f $R/fe/pom.xml -pl fe-core -am test -Dcheckstyle.skip=true -DfailIfNoTests=false --fail-at-end + ``` + - **状态**：残留 grep = 0 ✅ · 全反应堆 `test-compile` = BUILD SUCCESS ✅ · + `checkstyle:check` = 0 violations ✅ + - ⚠️ **完整 fe-core 套件未跑完**：跑到 **3h29m / 1232 个测试类**时由用户指示**主动终止** + （耗时问题另见 [`../fe-core-ut-runtime-problem.md`](../fe-core-ut-runtime-problem.md)）。 + 终止时**仅 1 个测试类失败**：`http.ForwardToMasterTest.testAddBeDropBe` + （`ClassCastException: JSONObject cannot be cast to JSONArray`）。 + **已用 stash 归因**：`git stash push -u -- fe/` 回到干净 HEAD 跑同一测试 → **一模一样地失败** + ⇒ **既有失败，与本改动无关**。其余 1231 个测试类全绿。 --- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
