This is an automated email from the ASF dual-hosted git repository.
CalvinKirs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 163a10a3c44 [improvement](fe) add fe_meta_auth_token for FE
meta-service internal HTTP auth (#65551)
163a10a3c44 is described below
commit 163a10a3c44bf298ad964ea84e360667ab3abd99
Author: Calvin Kirs <[email protected]>
AuthorDate: Thu Jul 16 12:06:28 2026 +0800
[improvement](fe) add fe_meta_auth_token for FE meta-service internal HTTP
auth (#65551)
### What problem does this PR solve?
The FE meta-service endpoints
(`image`/`role`/`check`/`put`/`journal_id`, etc.) authenticate callers
only by the `CLIENT_NODE_HOST`/`CLIENT_NODE_PORT` headers — i.e. whether
the claimed host:port is a registered FE. That is a plaintext claim with
no secret, so any host that knows a valid FE address can impersonate it.
### What this PR does
Add an optional cluster token `fe_meta_auth_token`:
- **Empty (default)** — behavior is unchanged: node-host check only.
Existing clusters and rolling upgrades are unaffected.
- **Set** — `checkFromValidFe` additionally requires the request to
carry a matching token header, **on top of** the existing node-host
check (additive, does not replace the host check).
The token is a static `fe.conf` item, so a scaling-out FE already holds
it before the bootstrap handshake — no chicken-and-egg with the token
that `/check` itself hands out. It must be identical on all FEs.
Additional hardening of the meta-service:
- `/put` rejects a port other than the FE HTTP port.
- `/dump` always checks the admin password.
- meta-helper logs header **names** only, never the token value.
### Tests
- `MetaServiceTest`: matching / missing / wrong token,
no-token-when-unconfigured, unknown-host-rejected-even-with-token,
`/put` port check, `/dump` auth.
- `HttpURLUtilTest`: token header emission (present/absent) and internal
URL building.
This supersedes #63782 (rebased onto latest master with a cleaner,
switch-free design).
---
.../main/java/org/apache/doris/common/Config.java | 14 +-
.../java/org/apache/doris/common/ConfigBase.java | 20 +-
.../java/org/apache/doris/common/ConfigTest.java | 58 ++++
.../org/apache/doris/common/util/HttpURLUtil.java | 10 +
.../org/apache/doris/httpv2/meta/MetaService.java | 77 +++++-
.../doris/httpv2/rest/manager/NodeAction.java | 13 +-
.../java/org/apache/doris/master/MetaHelper.java | 2 +-
.../apache/doris/common/util/HttpURLUtilTest.java | 59 +++++
.../apache/doris/httpv2/meta/MetaServiceTest.java | 295 +++++++++++++++++++++
.../suites/meta_action_p0/test_dump_image.groovy | 1 +
10 files changed, 532 insertions(+), 17 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 39f29b958d5..d59d4728256 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -491,7 +491,7 @@ public class Config extends ConfigBase {
+ "starts for the first time. You can also specify one."})
public static int cluster_id = -1;
- @ConfField(description = {"Cluster token used for internal
authentication."})
+ @ConfField(sensitive = true, description = {"Cluster token used for
internal authentication."})
public static String auth_token = "";
@ConfField(mutable = true, masterOnly = true,
@@ -825,6 +825,18 @@ public class Config extends ConfigBase {
// check token when download image file.
@ConfField public static boolean enable_token_check = true;
+ @ConfField(sensitive = true, description = {"Cluster token for FE
meta-service internal HTTP authentication. "
+ + "When set (non-empty), FE meta-service endpoints (such as
image/role/check/put/journal_id) "
+ + "additionally require the caller to present a matching token
header, on top of the existing "
+ + "node-host check. Empty (default) keeps the legacy behavior of
node-host check only, so "
+ + "existing clusters and rolling upgrades are unaffected. Must be
identical on all FEs and "
+ + "provisioned in fe.conf before enabling, otherwise FEs will
reject each other.",
+ "FE meta-service 内部 HTTP 鉴权使用的集群 token。设置(非空)后,meta-service 端点(如 "
+ + "image/role/check/put/journal_id)在原有 node-host 校验之上,额外要求调用方携带匹配的
token 头。"
+ + "为空(默认)时维持仅 node-host 校验的旧行为,存量集群与滚动升级不受影响。必须在所有 FE 上取值一致,"
+ + "并在启用前写入 fe.conf,否则 FE 之间会互相拒绝。"})
+ public static String fe_meta_auth_token = "";
+
/**
* Set to true if you deploy Palo using thirdparty deploy manager
* Valid options are:
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java
b/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java
index 192fdea7c8b..e58fea913d7 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java
@@ -52,6 +52,10 @@ public class ConfigBase {
boolean masterOnly() default false;
+ // If true, the value is a secret (e.g. a token or password) and is
masked in every
+ // config dump API (Config.dump / getConfigInfo), so it is never
returned in plaintext.
+ boolean sensitive() default false;
+
String comment() default "";
VariableAnnotation varType() default VariableAnnotation.NONE;
@@ -191,13 +195,26 @@ public class ConfigBase {
}
}
+ // Placeholder returned instead of a sensitive config's real value in any
dump API.
+ public static final String SENSITIVE_CONF_MASK = "********";
+
+ // Mask the value of a sensitive config (a non-empty secret) so it is
never dumped in plaintext.
+ // An empty value is left as-is: it reveals nothing and keeps "unset"
visible.
+ private static String maskIfSensitive(Field field, String value) {
+ ConfField anno = field.getAnnotation(ConfField.class);
+ if (anno != null && anno.sensitive() && !Strings.isNullOrEmpty(value))
{
+ return SENSITIVE_CONF_MASK;
+ }
+ return value;
+ }
+
public static HashMap<String, String> dump() {
HashMap<String, String> map = new HashMap<>();
Field[] fields = confClass.getFields();
for (Field f : fields) {
ConfField anno = f.getAnnotation(ConfField.class);
if (anno != null) {
- map.put(f.getName(), getConfValue(f));
+ map.put(f.getName(), maskIfSensitive(f, getConfValue(f)));
}
}
return map;
@@ -441,6 +458,7 @@ public class ConfigBase {
if (confKey.equals("sys_log_dir") &&
Strings.isNullOrEmpty(value)) {
value = System.getenv("DORIS_HOME") + "/log";
}
+ value = maskIfSensitive(f, value);
config.add(value);
config.add(f.getType().getSimpleName());
config.add(String.valueOf(confField.mutable()));
diff --git a/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java
b/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java
index 0d305298270..d48d7ae8ed3 100644
--- a/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java
+++ b/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java
@@ -23,6 +23,8 @@ import org.junit.Test;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
public class ConfigTest {
@BeforeClass
@@ -34,6 +36,62 @@ public class ConfigTest {
config.init(tempFile.toAbsolutePath().toString());
}
+ // A sensitive config (fe_meta_auth_token) must never be dumped in
plaintext by any config
+ // API: both Config.dump() and ConfigBase.getConfigInfo() return the mask
instead of the value.
+ @Test
+ public void testSensitiveConfigIsMaskedWhenSet() {
+ String old = Config.fe_meta_auth_token;
+ try {
+ Config.fe_meta_auth_token = "super-secret-token";
+
+ Map<String, String> dumped = ConfigBase.dump();
+ Assert.assertEquals(ConfigBase.SENSITIVE_CONF_MASK,
dumped.get("fe_meta_auth_token"));
+
+ String value = configInfoValue("fe_meta_auth_token");
+ Assert.assertEquals(ConfigBase.SENSITIVE_CONF_MASK, value);
+ } finally {
+ Config.fe_meta_auth_token = old;
+ }
+ }
+
+ // The legacy cluster secret auth_token is also marked sensitive, so it is
masked by every
+ // config dump API too (it leaks through /rest/v1/config/fe otherwise).
+ @Test
+ public void testAuthTokenIsMaskedWhenSet() {
+ String old = Config.auth_token;
+ try {
+ Config.auth_token = "super-secret-auth-token";
+
+ Assert.assertEquals(ConfigBase.SENSITIVE_CONF_MASK,
ConfigBase.dump().get("auth_token"));
+ Assert.assertEquals(ConfigBase.SENSITIVE_CONF_MASK,
configInfoValue("auth_token"));
+ } finally {
+ Config.auth_token = old;
+ }
+ }
+
+ // An empty sensitive config is left as-is (no secret to hide), so "unset"
stays visible.
+ @Test
+ public void testEmptySensitiveConfigIsNotMasked() {
+ String old = Config.fe_meta_auth_token;
+ try {
+ Config.fe_meta_auth_token = "";
+
+ Assert.assertEquals("",
ConfigBase.dump().get("fe_meta_auth_token"));
+ Assert.assertEquals("", configInfoValue("fe_meta_auth_token"));
+ } finally {
+ Config.fe_meta_auth_token = old;
+ }
+ }
+
+ private static String configInfoValue(String key) {
+ for (List<String> row : ConfigBase.getConfigInfo(null)) {
+ if (row.get(0).equals(key)) {
+ return row.get(1);
+ }
+ }
+ throw new IllegalStateException("config not found: " + key);
+ }
+
@Test
public void testSetEmptyArray() throws ConfigException {
ConfigBase.setMutableConfig("s3_load_endpoint_white_list", "a,b,c");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/HttpURLUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/HttpURLUtil.java
index feb88abbb1c..f1fa2f87d40 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/HttpURLUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/HttpURLUtil.java
@@ -20,8 +20,10 @@ package org.apache.doris.common.util;
import org.apache.doris.catalog.Env;
import org.apache.doris.cloud.security.SecurityChecker;
import org.apache.doris.common.Config;
+import org.apache.doris.httpv2.meta.MetaBaseAction;
import org.apache.doris.system.SystemInfoService.HostInfo;
+import com.google.common.base.Strings;
import com.google.common.collect.Maps;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
@@ -50,6 +52,10 @@ public class HttpURLUtil {
HostInfo selfNode = Env.getServingEnv().getSelfNode();
conn.setRequestProperty(Env.CLIENT_NODE_HOST_KEY,
selfNode.getHost());
conn.setRequestProperty(Env.CLIENT_NODE_PORT_KEY,
selfNode.getPort() + "");
+ String token = Config.fe_meta_auth_token;
+ if (!Strings.isNullOrEmpty(token)) {
+ conn.setRequestProperty(MetaBaseAction.TOKEN, token);
+ }
return conn;
} catch (Exception e) {
throw new IOException(e);
@@ -65,6 +71,10 @@ public class HttpURLUtil {
HostInfo selfNode = Env.getServingEnv().getSelfNode();
headers.put(Env.CLIENT_NODE_HOST_KEY, selfNode.getHost());
headers.put(Env.CLIENT_NODE_PORT_KEY, selfNode.getPort() + "");
+ String token = Config.fe_meta_auth_token;
+ if (!Strings.isNullOrEmpty(token)) {
+ headers.put(MetaBaseAction.TOKEN, token);
+ }
return headers;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java
index 79a34ad4d38..4a8acdce33d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java
@@ -20,11 +20,15 @@ package org.apache.doris.httpv2.meta;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
+import org.apache.doris.common.util.HttpURLUtil;
import org.apache.doris.common.util.NetUtils;
import org.apache.doris.ha.FrontendNodeType;
+import
org.apache.doris.httpv2.controller.BaseController.ActionAuthorizationInfo;
import org.apache.doris.httpv2.entity.ResponseEntityBuilder;
+import org.apache.doris.httpv2.exception.UnauthorizedException;
import org.apache.doris.httpv2.rest.RestBaseController;
import org.apache.doris.master.MetaHelper;
+import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.persist.MetaCleaner;
import org.apache.doris.persist.Storage;
import org.apache.doris.persist.StorageInfo;
@@ -55,33 +59,76 @@ public class MetaService extends RestBaseController {
private File imageDir = MetaHelper.getMasterImageDir();
- private boolean isFromValidFe(String clientHost, String clientPortStr) {
+ private Frontend getValidFe(String clientHost, String clientPortStr) {
Integer clientPort;
try {
clientPort = Integer.valueOf(clientPortStr);
} catch (Exception e) {
LOG.warn("get clientPort error. clientPortStr: {}", clientPortStr,
e.getMessage());
- return false;
+ return null;
}
Frontend fe = Env.getCurrentEnv().checkFeExist(clientHost, clientPort);
if (fe == null) {
LOG.warn("request is not from valid FE. client: {}, {}",
clientHost, clientPortStr);
- return false;
}
- return true;
+ return fe;
}
private void checkFromValidFe(HttpServletRequest request)
- throws InvalidClientException {
+ throws UnauthorizedException {
String clientHost = request.getHeader(Env.CLIENT_NODE_HOST_KEY);
String clientPort = request.getHeader(Env.CLIENT_NODE_PORT_KEY);
- if (!isFromValidFe(clientHost, clientPort)) {
- throw new InvalidClientException("invalid client host: " +
clientHost + ":" + clientPort
- + ", request from " + request.getRemoteHost());
+ Frontend fe = getValidFe(clientHost, clientPort);
+ if (fe == null) {
+ throw unauthorized(clientHost, clientPort, request);
+ }
+
+ // If a cluster meta auth token is configured, additionally require
the request to
+ // carry a matching token. An empty token keeps the legacy
node-host-only behavior,
+ // so existing clusters and rolling upgrades are unaffected.
+ String clusterToken = Config.fe_meta_auth_token;
+ if (!Strings.isNullOrEmpty(clusterToken)) {
+ String requestToken = request.getHeader(MetaBaseAction.TOKEN);
+ if (!clusterToken.equals(requestToken)) {
+ // Log a masked prefix of both tokens so token-rotation issues
are diagnosable
+ // (e.g. expected "abc***" vs actual "<empty>" means the peer
sent no token),
+ // while never revealing the full secret.
+ LOG.warn("reject meta request with invalid token. client: {},
{}, request from: {}, "
+ + "expected: {}, actual: {}",
+ clientHost, clientPort, request.getRemoteAddr(),
+ maskToken(clusterToken), maskToken(requestToken));
+ throw unauthorized(clientHost, clientPort, request);
+ }
}
}
+ private UnauthorizedException unauthorized(String clientHost, String
clientPort, HttpServletRequest request) {
+ return new UnauthorizedException("invalid client host: " + clientHost
+ ":" + clientPort
+ + ", request from " + request.getRemoteAddr());
+ }
+
+ // Minimum token length required before we reveal a masked prefix in logs.
Shorter tokens would
+ // leak too large a fraction of the secret, so they are hidden entirely
with only a length hint.
+ private static final int MIN_TOKEN_LEN_FOR_PREFIX = 8;
+ private static final int TOKEN_PREFIX_LEN = 3;
+
+ /**
+ * Masks a token for logging: reveals only a short leading prefix (e.g.
"abc***") so that a
+ * token mismatch is diagnosable during rotation, while never logging the
full secret. Empty
+ * tokens and tokens too short to safely show a prefix are hidden.
+ */
+ private static String maskToken(String token) {
+ if (Strings.isNullOrEmpty(token)) {
+ return "<empty>";
+ }
+ if (token.length() < MIN_TOKEN_LEN_FOR_PREFIX) {
+ // Too short to reveal any prefix without leaking a large fraction
of the secret.
+ return "<hidden, token length " + token.length() + " < " +
MIN_TOKEN_LEN_FOR_PREFIX + ">";
+ }
+ return token.substring(0, TOKEN_PREFIX_LEN) + "***";
+ }
+
@RequestMapping(path = "/image", method = RequestMethod.GET)
public Object image(HttpServletRequest request, HttpServletResponse
response) {
checkFromValidFe(request);
@@ -149,6 +196,12 @@ public class MetaService extends RestBaseController {
if (port < 0 || port > 65535) {
return ResponseEntityBuilder.badRequest("port is invalid. The port
number is between 0-65535");
}
+ // The master pushes image using HttpURLUtil.getHttpPort() (https_port
when enable_https=true,
+ // otherwise http_port), so the expected port must follow the same
rule to stay consistent.
+ int expectedPort = HttpURLUtil.getHttpPort();
+ if (port != expectedPort) {
+ return ResponseEntityBuilder.badRequest("port must be FE HTTP
port: " + expectedPort);
+ }
String versionStr = request.getParameter(VERSION);
if (Strings.isNullOrEmpty(versionStr)) {
@@ -245,9 +298,11 @@ public class MetaService extends RestBaseController {
@RequestMapping(value = "/dump", method = RequestMethod.GET)
public Object dump(HttpServletRequest request, HttpServletResponse
response) throws DdlException {
- if (Config.enable_all_http_auth) {
- executeCheckPassword(request, response);
- }
+ // /dump triggers a full metadata image dump (takes catalog/db/table
locks and writes an
+ // image file), so it must be ADMIN-gated. executeCheckPassword only
authenticates the
+ // caller; enforce the ADMIN privilege explicitly, matching other
metadata/debug operations.
+ ActionAuthorizationInfo authInfo = executeCheckPassword(request,
response);
+ checkGlobalAuth(authInfo.userIdentity, PrivPredicate.ADMIN);
/*
* Before dump, we acquired the catalog read lock and all databases'
read lock and all
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java
index 3a8dc73bbe2..fcb3a58a5d5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java
@@ -259,8 +259,13 @@ public class NodeAction extends RestBaseController {
*/
@RequestMapping(path = "/config", method = RequestMethod.GET)
public Object config(HttpServletRequest request, HttpServletResponse
response) {
- executeCheckPassword(request, response);
- checkDbAuth(ConnectContext.get().getCurrentUserIdentity(),
InfoSchemaDb.DATABASE_NAME, PrivPredicate.SELECT);
+ // This endpoint lists all FE config, matching the SQL "SHOW FRONTEND
CONFIG", which
+ // requires ADMIN. Use an unconditional ADMIN check: checkAdminAuth
only enforces the
+ // privilege when enable_all_http_auth is true, so it would be a no-op
by default.
+ // Sensitive config values (e.g. fe_meta_auth_token) are additionally
masked by ConfigBase,
+ // so they are never returned in plaintext even to an admin.
+ ActionAuthorizationInfo authInfo = executeCheckPassword(request,
response);
+ checkGlobalAuth(authInfo.userIdentity, PrivPredicate.ADMIN);
List<List<String>> configs = ConfigBase.getConfigInfo(null);
// Sort all configs by config key.
@@ -320,8 +325,10 @@ public class NodeAction extends RestBaseController {
public Object configurationInfo(HttpServletRequest request,
HttpServletResponse response,
@RequestParam(value = "type") String type,
@RequestBody(required = false) ConfigInfoRequestBody requestBody) {
+ // Reads FE/BE config via fan-out to the per-node config endpoints, so
it must be
+ // ADMIN-gated too. Unconditional check (see config() above for why
checkAdminAuth is not).
ActionAuthorizationInfo authInfo = executeCheckPassword(request,
response);
- checkAdminAuth(authInfo.userIdentity);
+ checkGlobalAuth(authInfo.userIdentity, PrivPredicate.ADMIN);
initHttpExecutor();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/MetaHelper.java
b/fe/fe-core/src/main/java/org/apache/doris/master/MetaHelper.java
index 5acec62ae59..73fceec8b39 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/MetaHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/MetaHelper.java
@@ -149,7 +149,7 @@ public class MetaHelper {
public static <T> ResponseBody doGet(String url, int timeout, Class<T>
clazz) throws IOException {
Map<String, String> headers = HttpURLUtil.getNodeIdentHeaders();
- LOG.info("meta helper, url: {}, timeout: {}, headers: {}", url,
timeout, headers);
+ LOG.info("meta helper, url: {}, timeout: {}, header names: {}", url,
timeout, headers.keySet());
String response = HttpUtils.doGet(url, headers, timeout);
try {
return parseResponse(response, clazz);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/util/HttpURLUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/util/HttpURLUtilTest.java
index 5bba968b49a..5d2099c5c61 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/util/HttpURLUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/HttpURLUtilTest.java
@@ -17,11 +17,19 @@
package org.apache.doris.common.util;
+import org.apache.doris.catalog.Env;
import org.apache.doris.common.Config;
+import org.apache.doris.httpv2.meta.MetaBaseAction;
+import org.apache.doris.system.SystemInfoService.HostInfo;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.net.HttpURLConnection;
+import java.util.Map;
public class HttpURLUtilTest {
@@ -30,6 +38,57 @@ public class HttpURLUtilTest {
Config.enable_https = false;
Config.http_port = 8030;
Config.https_port = 8050;
+ Config.fe_meta_auth_token = "";
+ }
+
+ @Test
+ public void testNodeIdentHeadersIncludeClusterToken() throws Exception {
+ Config.fe_meta_auth_token = "cluster-token";
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getSelfNode()).thenReturn(new HostInfo("127.0.0.1",
9010));
+
+ try (MockedStatic<Env> envStatic = Mockito.mockStatic(Env.class)) {
+ envStatic.when(Env::getServingEnv).thenReturn(env);
+
+ Map<String, String> headers = HttpURLUtil.getNodeIdentHeaders();
+
+ Assert.assertEquals("127.0.0.1",
headers.get(Env.CLIENT_NODE_HOST_KEY));
+ Assert.assertEquals("9010", headers.get(Env.CLIENT_NODE_PORT_KEY));
+ Assert.assertEquals("cluster-token",
headers.get(MetaBaseAction.TOKEN));
+ }
+ }
+
+ @Test
+ public void testNodeIdentHeadersOmitTokenWhenNotConfigured() throws
Exception {
+ Config.fe_meta_auth_token = "";
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getSelfNode()).thenReturn(new HostInfo("127.0.0.1",
9010));
+
+ try (MockedStatic<Env> envStatic = Mockito.mockStatic(Env.class)) {
+ envStatic.when(Env::getServingEnv).thenReturn(env);
+
+ Map<String, String> headers = HttpURLUtil.getNodeIdentHeaders();
+
+ Assert.assertEquals("127.0.0.1",
headers.get(Env.CLIENT_NODE_HOST_KEY));
+ Assert.assertFalse(headers.containsKey(MetaBaseAction.TOKEN));
+ }
+ }
+
+ @Test
+ public void testNodeIdentConnectionIncludesClusterToken() throws Exception
{
+ Config.fe_meta_auth_token = "cluster-token";
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getSelfNode()).thenReturn(new HostInfo("127.0.0.1",
9010));
+
+ try (MockedStatic<Env> envStatic = Mockito.mockStatic(Env.class)) {
+ envStatic.when(Env::getServingEnv).thenReturn(env);
+
+ HttpURLConnection connection =
HttpURLUtil.getConnectionWithNodeIdent("http://127.0.0.1:8030/info";);
+
+ Assert.assertEquals("127.0.0.1",
connection.getRequestProperty(Env.CLIENT_NODE_HOST_KEY));
+ Assert.assertEquals("9010",
connection.getRequestProperty(Env.CLIENT_NODE_PORT_KEY));
+ Assert.assertEquals("cluster-token",
connection.getRequestProperty(MetaBaseAction.TOKEN));
+ }
}
@Test
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/httpv2/meta/MetaServiceTest.java
b/fe/fe-core/src/test/java/org/apache/doris/httpv2/meta/MetaServiceTest.java
new file mode 100644
index 00000000000..90bb54de6b9
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/httpv2/meta/MetaServiceTest.java
@@ -0,0 +1,295 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.httpv2.meta;
+
+import org.apache.doris.analysis.UserIdentity;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.Config;
+import org.apache.doris.ha.FrontendNodeType;
+import org.apache.doris.httpv2.controller.BaseController;
+import org.apache.doris.httpv2.entity.ResponseBody;
+import org.apache.doris.httpv2.exception.UnauthorizedException;
+import org.apache.doris.httpv2.rest.RestApiStatusCode;
+import org.apache.doris.master.MetaHelper;
+import org.apache.doris.mysql.privilege.AccessControllerManager;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.persist.Storage;
+import org.apache.doris.system.Frontend;
+
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletResponse;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+import org.springframework.http.ResponseEntity;
+
+import java.io.File;
+import java.lang.reflect.Field;
+
+public class MetaServiceTest {
+ // Value configured via Config.fe_meta_auth_token -- the cluster meta auth
token.
+ private static final String META_TOKEN = "meta-auth-token";
+ // Token persisted in the local Storage, returned by the /check endpoint.
Intentionally
+ // different from META_TOKEN to show the two are independent.
+ private static final String STORAGE_TOKEN = "storage-token";
+ private static final String BAD_TOKEN = "bad-token";
+ private static final String FE_HOST = "127.0.0.1";
+ private static final int FE_EDIT_LOG_PORT = 9010;
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ private String oldMetaAuthToken;
+ private boolean oldEnableAllHttpAuth;
+ private int oldHttpPort;
+ private int oldHttpsPort;
+ private boolean oldEnableHttps;
+ private Env env;
+ private MockedStatic<Env> envStatic;
+
+ @Before
+ public void setUp() {
+ oldMetaAuthToken = Config.fe_meta_auth_token;
+ oldEnableAllHttpAuth = Config.enable_all_http_auth;
+ oldHttpPort = Config.http_port;
+ oldHttpsPort = Config.https_port;
+ oldEnableHttps = Config.enable_https;
+ // Token required by default; individual tests clear it to exercise
the legacy path.
+ Config.fe_meta_auth_token = META_TOKEN;
+ Config.enable_all_http_auth = false;
+ Config.http_port = 8030;
+ Config.https_port = 8050;
+ Config.enable_https = false;
+
+ env = Mockito.mock(Env.class);
+ envStatic = Mockito.mockStatic(Env.class);
+ envStatic.when(Env::getCurrentEnv).thenReturn(env);
+
+ Frontend frontend = new Frontend(FrontendNodeType.FOLLOWER, "fe1",
FE_HOST, FE_EDIT_LOG_PORT);
+ Mockito.when(env.checkFeExist(FE_HOST,
FE_EDIT_LOG_PORT)).thenReturn(frontend);
+
Mockito.when(env.getImageDir()).thenReturn(temporaryFolder.getRoot().getAbsolutePath());
+ }
+
+ @After
+ public void tearDown() {
+ Config.fe_meta_auth_token = oldMetaAuthToken;
+ Config.enable_all_http_auth = oldEnableAllHttpAuth;
+ Config.http_port = oldHttpPort;
+ Config.https_port = oldHttpsPort;
+ Config.enable_https = oldEnableHttps;
+ if (envStatic != null) {
+ envStatic.close();
+ }
+ }
+
+ // A matching token passes even when the request originates from a proxy
address
+ // (remoteAddr differs from the registered FE host).
+ @Test
+ public void testMatchingTokenAllowsMetaRequestFromProxyAddress() throws
Exception {
+ MetaService service = new MetaService();
+ HttpServletRequest request = newRequest("192.0.2.10", FE_HOST,
META_TOKEN);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+ Object result = service.role(request, response);
+
+ Assert.assertEquals(RestApiStatusCode.OK.code, responseCode(result));
+ Mockito.verify(response).setHeader("role",
FrontendNodeType.FOLLOWER.name());
+ }
+
+ // When no token is configured, the node-host check alone is sufficient.
This keeps
+ // existing clusters and rolling upgrades working (a peer that sends no
token is allowed).
+ @Test
+ public void testNoTokenRequiredWhenTokenNotConfigured() throws Exception {
+ Config.fe_meta_auth_token = "";
+ MetaService service = new MetaService();
+ HttpServletRequest request = newRequest("192.0.2.10", FE_HOST, null);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+ Object result = service.role(request, response);
+
+ Assert.assertEquals(RestApiStatusCode.OK.code, responseCode(result));
+ Mockito.verify(response).setHeader("role",
FrontendNodeType.FOLLOWER.name());
+ }
+
+ // /check returns the local Storage token once the request passes
authentication.
+ @Test
+ public void testCheckReturnsStorageTokenWhenAuthPasses() throws Exception {
+ Config.fe_meta_auth_token = "";
+ MetaService service = serviceWithImageDir();
+ HttpServletRequest request = newRequest("192.0.2.10", FE_HOST, null);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+ Object result = service.check(request, response);
+
+ Assert.assertEquals(RestApiStatusCode.OK.code, responseCode(result));
+ Mockito.verify(response).setHeader(MetaBaseAction.TOKEN,
STORAGE_TOKEN);
+ }
+
+ // A wrong token is rejected even though the node-host check would pass.
+ @Test
+ public void testWrongTokenRejected() {
+ MetaService service = new MetaService();
+ HttpServletRequest request = newRequest(FE_HOST, FE_HOST, BAD_TOKEN);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+ Assert.assertThrows(UnauthorizedException.class, () ->
service.role(request, response));
+ }
+
+ // A missing token is rejected when a cluster token is configured.
+ @Test
+ public void testMissingTokenRejectedWhenConfigured() {
+ MetaService service = new MetaService();
+ HttpServletRequest request = newRequest(FE_HOST, FE_HOST, null);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+ Assert.assertThrows(UnauthorizedException.class, () ->
service.role(request, response));
+ }
+
+ // The node-host check is always enforced: an unknown FE host is rejected
even with a
+ // matching token (the token check is additive, it does not replace the
host check).
+ @Test
+ public void testUnknownHostRejectedEvenWithValidToken() {
+ MetaService service = new MetaService();
+ HttpServletRequest request = newRequest("192.0.2.99", "192.0.2.99",
META_TOKEN);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+ Assert.assertThrows(UnauthorizedException.class, () ->
service.role(request, response));
+ }
+
+ @Test
+ public void testPutRejectsUnexpectedHttpPort() throws Exception {
+ MetaService service = new MetaService();
+ HttpServletRequest request = newRequest(FE_HOST, FE_HOST, META_TOKEN);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+ Mockito.when(request.getParameter("version")).thenReturn("100");
+
Mockito.when(request.getParameter("port")).thenReturn(Integer.toString(Config.http_port
+ 1));
+
+ try (MockedStatic<MetaHelper> metaHelper =
Mockito.mockStatic(MetaHelper.class)) {
+ File partialFile = temporaryFolder.newFile("image.100.part");
+ metaHelper.when(() -> MetaHelper.getFile(Mockito.anyString(),
Mockito.any(File.class)))
+ .thenReturn(partialFile);
+
+ Object result = service.put(request, response);
+
+ Assert.assertEquals(RestApiStatusCode.BAD_REQUEST.code,
responseCode(result));
+ metaHelper.verify(() ->
MetaHelper.getRemoteFile(Mockito.anyString(), Mockito.anyInt(),
+ Mockito.any(File.class)), Mockito.never());
+ }
+ }
+
+ // When HTTPS is enabled the master pushes image using https_port, so /put
must accept
+ // https_port. The expected port follows HttpURLUtil.getHttpPort(), not a
hard-coded http_port.
+ @Test
+ public void testPutAcceptsHttpsPortWhenHttpsEnabled() throws Exception {
+ Config.enable_https = true;
+ MetaService service = new MetaService();
+ HttpServletRequest request = newRequest(FE_HOST, FE_HOST, META_TOKEN);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+ Mockito.when(request.getParameter("version")).thenReturn("100");
+
Mockito.when(request.getParameter("port")).thenReturn(Integer.toString(Config.https_port));
+
+ try (MockedStatic<MetaHelper> metaHelper =
Mockito.mockStatic(MetaHelper.class)) {
+ File partialFile = temporaryFolder.newFile("image.100.part");
+ metaHelper.when(() -> MetaHelper.getFile(Mockito.anyString(),
Mockito.any(File.class)))
+ .thenReturn(partialFile);
+
+ Object result = service.put(request, response);
+
+ // Passes the port check and proceeds to fetch the remote image
(no BAD_REQUEST).
+ Assert.assertEquals(RestApiStatusCode.OK.code,
responseCode(result));
+ metaHelper.verify(() ->
MetaHelper.getRemoteFile(Mockito.anyString(), Mockito.anyInt(),
+ Mockito.any(File.class)), Mockito.times(1));
+ }
+ }
+
+ // /dump authenticates and then requires ADMIN. An authenticated admin
passes the privilege
+ // check and proceeds to dump.
+ @Test
+ public void testDumpAllowsAdmin() throws Exception {
+ MetaService service = Mockito.spy(new MetaService());
+ HttpServletRequest request = newRequest(FE_HOST, FE_HOST, META_TOKEN);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
Mockito.doReturn(authInfo(UserIdentity.ADMIN)).when(service).executeCheckPassword(request,
response);
+ AccessControllerManager accessManager =
Mockito.mock(AccessControllerManager.class);
+ Mockito.when(env.getAccessManager()).thenReturn(accessManager);
+ Mockito.when(accessManager.checkGlobalPriv(UserIdentity.ADMIN,
PrivPredicate.ADMIN)).thenReturn(true);
+ Mockito.when(env.dumpImage()).thenReturn(null);
+
+ service.dump(request, response);
+
+ Mockito.verify(service).executeCheckPassword(request, response);
+ Mockito.verify(env).dumpImage();
+ }
+
+ // An authenticated but non-admin user is rejected before any dump happens.
+ @Test
+ public void testDumpRejectsNonAdmin() throws Exception {
+ MetaService service = Mockito.spy(new MetaService());
+ HttpServletRequest request = newRequest(FE_HOST, FE_HOST, META_TOKEN);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+ UserIdentity nonAdmin =
UserIdentity.createAnalyzedUserIdentWithIp("user", "%");
+
Mockito.doReturn(authInfo(nonAdmin)).when(service).executeCheckPassword(request,
response);
+ AccessControllerManager accessManager =
Mockito.mock(AccessControllerManager.class);
+ Mockito.when(env.getAccessManager()).thenReturn(accessManager);
+ Mockito.when(accessManager.checkGlobalPriv(nonAdmin,
PrivPredicate.ADMIN)).thenReturn(false);
+
+ Assert.assertThrows(UnauthorizedException.class, () ->
service.dump(request, response));
+ Mockito.verify(env, Mockito.never()).dumpImage();
+ }
+
+ private static BaseController.ActionAuthorizationInfo
authInfo(UserIdentity userIdentity) {
+ BaseController.ActionAuthorizationInfo info = new
BaseController.ActionAuthorizationInfo();
+ info.userIdentity = userIdentity;
+ return info;
+ }
+
+ private MetaService serviceWithImageDir() throws Exception {
+ File imageDir = temporaryFolder.newFolder("image");
+ Storage storage = new Storage(12345, STORAGE_TOKEN,
imageDir.getAbsolutePath());
+ storage.writeClusterIdAndToken();
+
+ MetaService service = new MetaService();
+ Field imageDirField = MetaService.class.getDeclaredField("imageDir");
+ imageDirField.setAccessible(true);
+ imageDirField.set(service, imageDir);
+ return service;
+ }
+
+ private HttpServletRequest newRequest(String remoteAddr, String
clientHost, String token) {
+ HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+
Mockito.when(request.getHeader(Env.CLIENT_NODE_HOST_KEY)).thenReturn(clientHost);
+
Mockito.when(request.getHeader(Env.CLIENT_NODE_PORT_KEY)).thenReturn(Integer.toString(FE_EDIT_LOG_PORT));
+
Mockito.when(request.getHeader(MetaBaseAction.TOKEN)).thenReturn(token);
+ Mockito.when(request.getRemoteAddr()).thenReturn(remoteAddr);
+ Mockito.when(request.getRemoteHost()).thenReturn(remoteAddr);
+ Mockito.when(request.getParameter("host")).thenReturn(clientHost);
+
Mockito.when(request.getParameter("port")).thenReturn(Integer.toString(FE_EDIT_LOG_PORT));
+ return request;
+ }
+
+ private int responseCode(Object result) {
+ ResponseEntity<?> responseEntity = (ResponseEntity<?>) result;
+ ResponseBody<?> body = (ResponseBody<?>) responseEntity.getBody();
+ return body.getCode();
+ }
+}
diff --git a/regression-test/suites/meta_action_p0/test_dump_image.groovy
b/regression-test/suites/meta_action_p0/test_dump_image.groovy
index 4500503ae42..6102dfff235 100644
--- a/regression-test/suites/meta_action_p0/test_dump_image.groovy
+++ b/regression-test/suites/meta_action_p0/test_dump_image.groovy
@@ -17,6 +17,7 @@
suite("test_dump_image", "nonConcurrent") {
httpTest {
+ basicAuthorization "${context.config.jdbcUser}",
"${context.config.jdbcPassword}"
endpoint context.config.feHttpAddress
uri "/dump"
op "get"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
