This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-4.2 in repository https://gitbox.apache.org/repos/asf/doris.git
commit e2f912099762b45531d1aca745be891256e73935 Author: Calvin Kirs <[email protected]> AuthorDate: Tue Oct 13 09:23:34 2026 +0800 branch-4.1: [feature](fe) Improve MySQL 9 client and Connector/J compatibility (#68824) (#68944) Related PR: #68824 --- .../main/java/org/apache/doris/common/Config.java | 12 ++ .../java/org/apache/doris/common/ConfigBase.java | 28 ++- .../java/org/apache/doris/common/ConfigTest.java | 68 ++++++ .../doris/mysql/CachingSha2PasswordExchange.java | 236 +++++++++++++++++++++ .../org/apache/doris/mysql/MysqlCapability.java | 10 +- .../java/org/apache/doris/mysql/MysqlChannel.java | 4 + .../password/NativePasswordResolver.java | 45 +++- .../nereids/trees/expressions/literal/Literal.java | 36 ++++ .../java/org/apache/doris/qe/GlobalVariable.java | 6 + .../org/apache/doris/qe/MysqlConnectProcessor.java | 40 +++- .../apache/doris/mysql/MysqlCapabilityTest.java | 20 +- .../doris/mysql/MysqlHandshakePacketTest.java | 9 +- .../org/apache/doris/mysql/MysqlProtoTest.java | 41 ++++ .../password/NativePasswordResolverTest.java | 233 ++++++++++++++++++++ .../expressions/literal/TimeV2LiteralTest.java | 110 ++++++++++ .../plans/commands/ShowConfigCommandTest.java | 7 +- .../java/org/apache/doris/qe/VariableMgrTest.java | 12 ++ 17 files changed, 895 insertions(+), 22 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index e5b7ca1b114..202a0ebcb9b 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2659,6 +2659,18 @@ public class Config extends ConfigBase { @ConfField(mutable = false, masterOnly = false) public static boolean ssl_force_client_auth = false; + /** + * Which MySQL clients that announce caching_sha2_password are authenticated with it (through its + * full-authentication exchange, the password is still checked against the stored native hash): + * "auto" - clients announcing caching_sha2_password with _client_version 9 or later + * (such as mysql / mysqldump 9.x); every other client is switched to mysql_native_password + * as before. "all" - every client announcing caching_sha2_password. "none" - no client; + * every client is switched to mysql_native_password, the behavior before this option. + */ + @ConfField(mutable = true, masterOnly = false, options = {"auto", "all", "none"}, + callback = OptionsConfHandler.class) + public static String mysql_caching_sha2_password_clients = "auto"; + /** * ssl connection needs to authenticate client's certificate store type. */ diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java b/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java index 177603f0fc2..e828f5bfcec 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java @@ -85,6 +85,25 @@ public class ConfigBase { } } + // The value must be one of the annotation's options; stored as the option's own spelling. + public static class OptionsConfHandler implements ConfHandler { + @Override + public void handle(Field field, String confVal) throws Exception { + setConfigField(field, checkedOption(field, confVal)); + } + + // Pure validation shared by startup loading and dynamic updates. + private static String checkedOption(Field field, String confVal) throws ConfigException { + for (String option : field.getAnnotation(ConfField.class).options()) { + if (option.equalsIgnoreCase(confVal.trim())) { + return option; + } + } + throw new ConfigException("Config '" + field.getName() + "' must be one of " + + String.join(", ", field.getAnnotation(ConfField.class).options())); + } + } + static class CommaSeparatedIntersectConfHandler implements ConfHandler { @Override public void handle(Field field, String newVal) throws Exception { @@ -257,11 +276,18 @@ public class ConfigBase { // ensure that field has property string String confKey = f.getName(); String confVal = props.getProperty(confKey, props.getProperty(anno.varType().getPrefix() + confKey)); - if (Strings.isNullOrEmpty(confVal)) { + if (confVal == null) { continue; } try { + // Runtime callbacks can have side effects; only options validation is safe here. + if (anno.callback() == OptionsConfHandler.class) { + confVal = OptionsConfHandler.checkedOption(f, confVal); + } + if (confVal.isEmpty()) { + continue; + } setConfigField(f, confVal); } catch (Exception e) { String msg = String.format("Failed to set config, name: %s, value: %s", f.getName(), confVal); diff --git a/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java b/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java index f7ab2deea2e..9f1029bdaa2 100644 --- a/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java +++ b/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java @@ -20,6 +20,7 @@ package org.apache.doris.common; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.jupiter.api.Assertions; import java.nio.file.Files; import java.nio.file.Path; @@ -92,6 +93,73 @@ public class ConfigTest { throw new IllegalStateException("config not found: " + key); } + @Test + public void testOptionsConfigAcceptsItsOptionsOnly() throws ConfigException { + String original = Config.mysql_caching_sha2_password_clients; + try { + ConfigBase.setMutableConfig("mysql_caching_sha2_password_clients", "ALL"); + Assertions.assertEquals("all", Config.mysql_caching_sha2_password_clients); + Assertions.assertThrows(ConfigException.class, + () -> ConfigBase.setMutableConfig("mysql_caching_sha2_password_clients", "bogus")); + Assertions.assertEquals("all", Config.mysql_caching_sha2_password_clients); + } finally { + Config.mysql_caching_sha2_password_clients = original; + } + } + + @Test + public void testOptionsConfigIsValidatedDuringStartup() throws Exception { + String original = Config.mysql_caching_sha2_password_clients; + int originalInterval = Config.cloud_warm_up_job_scheduler_interval_millisecond; + try { + Config config = new Config(); + Path file = configFile("mysql_caching_sha2_password_clients= ALL \n" + + "cloud_warm_up_job_scheduler_interval_millisecond=0\n"); + config.init(file.toString()); + Assertions.assertEquals("all", Config.mysql_caching_sha2_password_clients); + // This unrelated callback rejects zero at runtime; startup must not dispatch it. + Assertions.assertEquals(0, Config.cloud_warm_up_job_scheduler_interval_millisecond); + config.init(configFile("").toString()); + Assertions.assertEquals("all", Config.mysql_caching_sha2_password_clients); + for (String value : new String[] {"nnoe", "", " "}) { + IllegalArgumentException error = Assertions.assertThrows(IllegalArgumentException.class, + () -> config.init(configFile("mysql_caching_sha2_password_clients=" + value + "\n") + .toString())); + Assertions.assertInstanceOf(ConfigException.class, error.getCause()); + Assertions.assertEquals("all", Config.mysql_caching_sha2_password_clients); + } + } finally { + Config.mysql_caching_sha2_password_clients = original; + Config.cloud_warm_up_job_scheduler_interval_millisecond = originalInterval; + } + } + + @Test + public void testOptionsConfigIsValidatedInCustomStartupFile() throws Exception { + String original = Config.mysql_caching_sha2_password_clients; + try { + Config config = new Config(); + config.init(configFile("mysql_caching_sha2_password_clients=auto\n").toString()); + Path custom = configFile("mysql_caching_sha2_password_clients= NONE \n"); + config.initCustom(custom.toString()); + Assertions.assertEquals("none", Config.mysql_caching_sha2_password_clients); + Files.writeString(custom, "mysql_caching_sha2_password_clients=nnoe\n"); + IllegalArgumentException error = Assertions.assertThrows(IllegalArgumentException.class, + () -> config.initCustom(custom.toString())); + Assertions.assertInstanceOf(ConfigException.class, error.getCause()); + Assertions.assertEquals("none", Config.mysql_caching_sha2_password_clients); + } finally { + Config.mysql_caching_sha2_password_clients = original; + } + } + + private static Path configFile(String contents) throws Exception { + Path file = Files.createTempFile("fe_ut_options_", ".conf"); + file.toFile().deleteOnExit(); + Files.writeString(file, contents); + return file; + } + @Test public void testSetEmptyArray() throws ConfigException { ConfigBase.setMutableConfig("mysql_compat_var_whitelist", "a,b,c"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/CachingSha2PasswordExchange.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/CachingSha2PasswordExchange.java new file mode 100644 index 00000000000..df854ddf291 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/CachingSha2PasswordExchange.java @@ -0,0 +1,236 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mysql; + +import org.apache.doris.common.Config; + +import com.google.common.base.Strings; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.util.Base64; +import java.util.Map; +import javax.crypto.Cipher; + +/** + * Server side of the caching_sha2_password "full authentication" exchange, which yields the + * client's plaintext password. Doris keeps only the mysql_native_password hash, so the fast path + * of the plugin (a SHA-256 scramble) can never be verified here; every client that authenticates + * with this plugin is taken through full authentication and the password is then checked the + * native way by the caller. + * + * Why this exists: MySQL 9 moved mysql_native_password from its client library's built-in plugins + * to a dynamically loadable plugin. A client without that plugin fails when asked to switch to it, + * before Doris sees a password. Such a client announces caching_sha2_password in its handshake + * response, with an empty auth response and the connection attribute _client_version. + * + * The exchange, after the client's handshake response: + * <pre> + * server: AuthSwitchRequest(caching_sha2_password, nonce) (the handshake nonce is reused) + * client: 32-byte scramble, or empty for an empty password + * server: AuthMoreData 0x04 (perform full authentication) + * client: over TLS, the NUL-terminated plaintext password; + * otherwise 0x02 to ask for the server's RSA public key, or the encrypted password + * right away when it already holds the key (--server-public-key-path) + * server: AuthMoreData + PEM public key, when asked + * client: RSA(password XOR nonce, NUL-terminated), OAEP or PKCS#1 padded + * </pre> + * The RSA key pair lives in this process only; a client fetches the public key on every + * connection, so frontends need not share one. + */ +public class CachingSha2PasswordExchange { + private static final Logger LOG = LogManager.getLogger(CachingSha2PasswordExchange.class); + + public static final String PLUGIN_NAME = "caching_sha2_password"; + public static final String CLIENT_VERSION_ATTR = "_client_version"; + + private static final int AUTH_SWITCH_REQUEST = 0xfe; + private static final int AUTH_MORE_DATA = 0x01; + private static final int PERFORM_FULL_AUTHENTICATION = 0x04; + private static final int REQUEST_PUBLIC_KEY = 0x02; + private static final int SCRAMBLE_LENGTH = 32; + // libmysqlclient always encrypts with OAEP. Connector/J picks the padding from the server + // version it was told, OAEP from 8.0.5 on and PKCS#1 v1.5 before, and Doris reports itself as + // 5.7, so both are accepted; the padding is only told apart by trying. + private static final String[] RSA_TRANSFORMATIONS = {"RSA/ECB/OAEPWithSHA-1AndMGF1Padding", + "RSA/ECB/PKCS1Padding"}; + + private static volatile KeyPair keyPair; + + /** + * Whether a client that announced this plugin is served with it instead of being switched to + * mysql_native_password, per Config.mysql_caching_sha2_password_clients: "auto" serves + * clients reporting _client_version 9 or later; "all" serves every client that asks for the + * plugin; "none" keeps switching every client to the native plugin. + */ + public static boolean serves(String pluginName, Map<String, String> connectAttributes) { + if (!PLUGIN_NAME.equals(pluginName)) { + return false; + } + String mode = Config.mysql_caching_sha2_password_clients; + if ("all".equalsIgnoreCase(mode)) { + return true; + } + if ("none".equalsIgnoreCase(mode)) { + return false; + } + return isClientVersion9OrLater(connectAttributes.get(CLIENT_VERSION_ATTR)); + } + + // Only the >= 9 decision matters. Stop once reached so a long numeric prefix cannot overflow. + static boolean isClientVersion9OrLater(String clientVersion) { + if (Strings.isNullOrEmpty(clientVersion)) { + return false; + } + int major = 0; + for (int i = 0; i < clientVersion.length(); i++) { + char digit = clientVersion.charAt(i); + if (!Character.isDigit(digit)) { + break; + } + major = major * 10 + Character.digit(digit, 10); + if (major >= 9) { + return true; + } + } + return false; + } + + /** The client answered with something that is not part of the exchange. */ + public static class Rejected extends Exception { + Rejected(String message) { + super(message); + } + } + + /** + * Runs the exchange on the channel and returns the plaintext password, or null when the + * client closed the connection. + */ + public static String exchange(MysqlChannel channel, MysqlSerializer serializer, byte[] nonce, + boolean ssl) throws IOException, Rejected { + serializer.reset(); + serializer.writeInt1(AUTH_SWITCH_REQUEST); + serializer.writeNulTerminateString(PLUGIN_NAME); + serializer.writeBytes(nonce); + serializer.writeInt1(0); + channel.sendAndFlush(serializer.toByteBuffer()); + + ByteBuffer scramble = channel.fetchOnePacket(); + if (scramble == null) { + return null; + } + if (isEmptyPassword(scramble)) { + return ""; + } + if (scramble.remaining() != SCRAMBLE_LENGTH) { + throw new Rejected("unexpected scramble length " + scramble.remaining()); + } + + serializer.reset(); + serializer.writeInt1(AUTH_MORE_DATA); + serializer.writeInt1(PERFORM_FULL_AUTHENTICATION); + channel.sendAndFlush(serializer.toByteBuffer()); + + ByteBuffer response = channel.fetchOnePacket(); + if (response == null) { + return null; + } + if (ssl) { + LOG.debug("caching_sha2_password: full authentication over TLS"); + return new String(MysqlProto.readNulTerminateString(response), StandardCharsets.UTF_8); + } + if (response.remaining() == 1 && (response.get(response.position()) & 0xff) == REQUEST_PUBLIC_KEY) { + serializer.reset(); + serializer.writeInt1(AUTH_MORE_DATA); + serializer.writeBytes(publicKeyPem().getBytes(StandardCharsets.US_ASCII)); + channel.sendAndFlush(serializer.toByteBuffer()); + response = channel.fetchOnePacket(); + if (response == null) { + return null; + } + } + byte[] encrypted = new byte[response.remaining()]; + response.get(encrypted); + LOG.debug("caching_sha2_password: full authentication over RSA"); + return decryptPassword(encrypted, nonce); + } + + // An empty password is answered with no scramble: libmysqlclient sends a single NUL byte, the + // empty string with its terminator; nothing more is fetched from such a client. + private static boolean isEmptyPassword(ByteBuffer scramble) { + return scramble.remaining() == 0 + || (scramble.remaining() == 1 && scramble.get(scramble.position()) == 0); + } + + // RSA ciphertext of (password + NUL) XOR nonce, as libmysqlclient and Connector/J send it + static String decryptPassword(byte[] encrypted, byte[] nonce) throws Rejected { + byte[] xored = null; + for (String transformation : RSA_TRANSFORMATIONS) { + try { + Cipher cipher = Cipher.getInstance(transformation); + cipher.init(Cipher.DECRYPT_MODE, keyPair().getPrivate()); + xored = cipher.doFinal(encrypted); + break; + } catch (GeneralSecurityException e) { + LOG.debug("caching_sha2_password: not {}: {}", transformation, e.toString()); + } + } + if (xored == null) { + throw new Rejected("cannot decrypt the password with either RSA padding"); + } + byte[] plain = new byte[xored.length]; + for (int i = 0; i < xored.length; i++) { + plain[i] = (byte) (xored[i] ^ nonce[i % nonce.length]); + } + int end = plain.length; + if (end > 0 && plain[end - 1] == 0) { + end--; + } + return new String(plain, 0, end, StandardCharsets.UTF_8); + } + + public static String publicKeyPem() { + String base64 = Base64.getMimeEncoder(64, "\n".getBytes(StandardCharsets.US_ASCII)) + .encodeToString(keyPair().getPublic().getEncoded()); + return "-----BEGIN PUBLIC KEY-----\n" + base64 + "\n-----END PUBLIC KEY-----\n"; + } + + private static KeyPair keyPair() { + if (keyPair == null) { + synchronized (CachingSha2PasswordExchange.class) { + if (keyPair == null) { + try { + KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA"); + generator.initialize(2048); + keyPair = generator.generateKeyPair(); + } catch (GeneralSecurityException e) { + throw new IllegalStateException("cannot generate the RSA key pair", e); + } + } + } + } + return keyPair; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlCapability.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlCapability.java index 785d71520fd..d0e03d3c973 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlCapability.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlCapability.java @@ -72,12 +72,20 @@ public class MysqlCapability { private static final EnumSet<Flag> FLAG_SET = EnumSet.allOf(Flag.class); + // What the server advertises in the handshake, and its side of the negotiation: a connection + // gets the intersection of this set with what the client asks for. The multi-statement flags + // are advertised as MySQL does because Connector/J 8 and 9 mask the flags they ask for with the + // advertised ones: without them a driver with allowMultiQueries=true never asks for + // CLIENT_MULTI_STATEMENTS, and gets only the last statement's response of a multi-statement + // request (see ConnectProcessor.handleQuery). private static final int DEFAULT_FLAGS = Flag.CLIENT_PROTOCOL_41.getFlagBit() | Flag.CLIENT_CONNECT_WITH_DB.getFlagBit() | Flag.CLIENT_SECURE_CONNECTION.getFlagBit() | Flag.CLIENT_PLUGIN_AUTH.getFlagBit() | Flag.CLIENT_CONNECT_ATTRS.getFlagBit() | Flag.CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA.getFlagBit() | Flag.CLIENT_LOCAL_FILES.getFlagBit() | Flag.CLIENT_LONG_FLAG.getFlagBit() - | Flag.CLIENT_DEPRECATE_EOF.getFlagBit(); + | Flag.CLIENT_DEPRECATE_EOF.getFlagBit() + | Flag.CLIENT_MULTI_STATEMENTS.getFlagBit() | Flag.CLIENT_MULTI_RESULTS.getFlagBit() + | Flag.CLIENT_PS_MULTI_RESULTS.getFlagBit(); private static final int SSL_FLAGS = DEFAULT_FLAGS | Flag.CLIENT_SSL.getFlagBit(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlChannel.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlChannel.java index fc2573fd564..73fd522c22b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlChannel.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlChannel.java @@ -157,6 +157,10 @@ public class MysqlChannel implements BytesChannel { encryptNetData = ByteBuffer.allocate(sslEngine.getSession().getPacketBufferSize() * 2); } + public boolean isSslMode() { + return isSslMode; + } + public void setSslMode(boolean sslMode) { isSslMode = sslMode; if (isSslMode) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/password/NativePasswordResolver.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/password/NativePasswordResolver.java index 1e7a90c1017..e934e0c2d61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/password/NativePasswordResolver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/password/NativePasswordResolver.java @@ -19,19 +19,26 @@ package org.apache.doris.mysql.authenticate.password; import org.apache.doris.authentication.CredentialType; import org.apache.doris.common.Config; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.mysql.CachingSha2PasswordExchange; import org.apache.doris.mysql.MysqlAuthPacket; import org.apache.doris.mysql.MysqlChannel; import org.apache.doris.mysql.MysqlHandshakePacket; +import org.apache.doris.mysql.MysqlPassword; import org.apache.doris.mysql.MysqlProto; import org.apache.doris.mysql.MysqlSerializer; import org.apache.doris.mysql.authenticate.AuthenticateRequest; import org.apache.doris.qe.ConnectContext; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.io.IOException; import java.nio.ByteBuffer; import java.util.Optional; public class NativePasswordResolver implements PasswordResolver { + private static final Logger LOG = LogManager.getLogger(NativePasswordResolver.class); public static final String MYSQL_RANDOM_STRING_PROPERTY = "mysql.random_string"; @Override @@ -48,8 +55,35 @@ public class NativePasswordResolver implements PasswordResolver { // which Doris is using now. // Note: Check the authPacket whether support plugin auth firstly, // before we check AuthPlugin between doris and client to compatible with older version: like mysql 5.1 + // NOTE: when we behind proxy, we need random string sent by proxy. + byte[] randomString = handshakePacket.getAuthPluginData(); + if (Config.proxy_auth_enable && authPacket.getRandomString() != null) { + randomString = authPacket.getRandomString(); + } if (authPacket.getCapability().isPluginAuth() && !handshakePacket.checkAuthPluginSameAsDoris(authPacket.getPluginName())) { + if (CachingSha2PasswordExchange.serves(authPacket.getPluginName(), authPacket.getConnectAttributes())) { + // Complete caching_sha2_password full authentication for this client, then check + // the plaintext the native way so stored hashes, the proxy nonce and every check + // below stay as they are. + String plainPassword; + try { + plainPassword = CachingSha2PasswordExchange.exchange(channel, serializer, + handshakePacket.getAuthPluginData(), channel.isSslMode()); + } catch (CachingSha2PasswordExchange.Rejected e) { + LOG.warn("caching_sha2_password exchange rejected for client {}: {}", channel.getRemoteIp(), + e.getMessage()); + context.getState().setError(ErrorCode.ERR_NOT_SUPPORTED_AUTH_MODE, + ErrorCode.ERR_NOT_SUPPORTED_AUTH_MODE.formatErrorMsg()); + MysqlProto.sendResponsePacket(context); + return Optional.empty(); + } + if (plainPassword == null) { + // the client went away, as with the native switch below + return Optional.empty(); + } + return Optional.of(new NativePassword(nativeScramble(plainPassword, randomString), randomString)); + } // 1. clear the serializer serializer.reset(); // 2. build the auth switch request and send to the client @@ -66,14 +100,15 @@ public class NativePasswordResolver implements PasswordResolver { authResponse = MysqlProto.readEofString(authSwitchResponse); } - // NOTE: when we behind proxy, we need random string sent by proxy. - byte[] randomString = handshakePacket.getAuthPluginData(); - if (Config.proxy_auth_enable && authPacket.getRandomString() != null) { - randomString = authPacket.getRandomString(); - } return Optional.of(new NativePassword(authResponse, randomString)); } + // What a mysql_native_password client would have sent for this password: an empty auth + // response for an empty password, the SHA-1 scramble over the nonce otherwise. + public static byte[] nativeScramble(String plainPassword, byte[] nonce) { + return plainPassword.isEmpty() ? new byte[0] : MysqlPassword.scramble(nonce, plainPassword); + } + @Override public Optional<AuthenticateRequest> resolveAuthenticateRequest(String userName, ConnectContext context, MysqlChannel channel, MysqlSerializer serializer, MysqlAuthPacket authPacket, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java index 0260ba2e391..57abdd2fa12 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java @@ -703,6 +703,10 @@ public abstract class Literal extends Expression implements LeafExpression { case MYSQL_TYPE_TIMESTAMP2: literal = handleDateTimeLiteral(data); break; + case MYSQL_TYPE_TIME: + case MYSQL_TYPE_TIME2: + literal = handleTimeLiteral(data); + break; case MYSQL_TYPE_STRING: case MYSQL_TYPE_VARSTRING: literal = handleStringLiteral(data); @@ -780,6 +784,38 @@ public abstract class Literal extends Expression implements LeafExpression { } } + // Protocol::MYSQL_TYPE_TIME: length 0 is 00:00:00; 8 carries is_negative, days, hours, + // minutes, seconds; 12 adds microseconds. + private static Literal handleTimeLiteral(ByteBuffer data) throws AnalysisException { + if (!data.hasRemaining()) { + throw new AnalysisException("Missing binary TIME parameter length"); + } + int len = MysqlProto.readInt1(data); + if (len != 0 && len != 8 && len != 12) { + throw new AnalysisException("Invalid binary TIME parameter length: " + len); + } + if (data.remaining() < len) { + throw new AnalysisException("Truncated binary TIME parameter"); + } + if (len == 0) { + return new TimeV2Literal(0, 0, 0, 0, 6, false); + } + int sign = MysqlProto.readInt1(data); + if (sign != 0 && sign != 1) { + throw new AnalysisException("Invalid binary TIME parameter sign: " + sign); + } + long days = Integer.toUnsignedLong(data.getInt()); + long hours = days * 24 + MysqlProto.readInt1(data); + int minute = MysqlProto.readInt1(data); + int second = MysqlProto.readInt1(data); + int microsecond = len == 12 ? data.getInt() : 0; + try { + return new TimeV2Literal(Math.toIntExact(hours), minute, second, microsecond, 6, sign == 1); + } catch (ArithmeticException e) { + throw new AnalysisException("Binary TIME parameter is out of range", e); + } + } + private static Literal handleStringLiteral(ByteBuffer data) { int strLen = getParmLen(data); strLen = Math.min(strLen, data.remaining()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java index d304c78ded4..dd092bacc80 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java @@ -46,6 +46,7 @@ public final class GlobalVariable { public static final String VERSION = "version"; public static final String LOWER_CASE_TABLE_NAMES = "lower_case_table_names"; public static final String LICENSE = "license"; + public static final String AUTHENTICATION_POLICY = "authentication_policy"; public static final String LANGUAGE = "language"; public static final String INIT_CONNECT = "init_connect"; public static final String SYSTEM_TIME_ZONE = "system_time_zone"; @@ -109,6 +110,11 @@ public final class GlobalVariable { @VariableMgr.VarAttr(name = LICENSE, flag = VariableMgr.READ_ONLY) public static String license = "Apache License, Version 2.0"; + // MySQL 8.0.27+ reports the multi-factor authentication policy here and tools built for + // MySQL 8.4+ read it. Doris authenticates with a single factor, which is the MySQL default. + @VariableMgr.VarAttr(name = AUTHENTICATION_POLICY, flag = VariableMgr.READ_ONLY) + public static String authenticationPolicy = "*,,"; + @VariableMgr.VarAttr(name = LANGUAGE, flag = VariableMgr.READ_ONLY) public static String language = "/palo/share/english/"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index 150bf613356..1f46681f045 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -27,11 +27,13 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.Pair; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.mysql.CachingSha2PasswordExchange; import org.apache.doris.mysql.MysqlChannel; import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.mysql.MysqlHandshakePacket; import org.apache.doris.mysql.MysqlProto; import org.apache.doris.mysql.MysqlSerializer; +import org.apache.doris.mysql.authenticate.password.NativePasswordResolver; import org.apache.doris.mysql.privilege.Auth; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.glue.LogicalPlanAdapter; @@ -346,16 +348,34 @@ public class MysqlConnectProcessor extends ConnectProcessor { if (!MysqlHandshakePacket.AUTH_PLUGIN_NAME.equals(authPluginName)) { MysqlChannel channel = ctx.mysqlChannel; MysqlSerializer serializer = MysqlSerializer.newInstance(); - serializer.writeInt1((byte) 0xfe); - serializer.writeNulTerminateString(MysqlHandshakePacket.AUTH_PLUGIN_NAME); - serializer.writeBytes(authPluginData); - serializer.writeInt1(0); - channel.sendAndFlush(serializer.toByteBuffer()); - // Server receive auth switch response packet from client. - ByteBuffer authSwitchResponse = channel.fetchOnePacket(); - int length = authSwitchResponse.limit(); - password = new byte[length]; - System.arraycopy(authSwitchResponse.array(), 0, password, 0, length); + if (CachingSha2PasswordExchange.serves(authPluginName, ctx.getConnectAttributes())) { + // Same client class as at login (see NativePasswordResolver): full authentication, + // then the plaintext is checked the native way below. + String plainPassword; + try { + plainPassword = CachingSha2PasswordExchange.exchange(channel, serializer, + authPluginData, channel.isSslMode()); + } catch (CachingSha2PasswordExchange.Rejected e) { + LOG.warn("caching_sha2_password exchange rejected on COM_CHANGE_USER: {}", e.getMessage()); + plainPassword = null; + } + if (plainPassword == null) { + ctx.getState().setError(ErrorCode.ERR_ACCESS_DENIED_ERROR, "Authentication failed."); + return; + } + password = NativePasswordResolver.nativeScramble(plainPassword, authPluginData); + } else { + serializer.writeInt1((byte) 0xfe); + serializer.writeNulTerminateString(MysqlHandshakePacket.AUTH_PLUGIN_NAME); + serializer.writeBytes(authPluginData); + serializer.writeInt1(0); + channel.sendAndFlush(serializer.toByteBuffer()); + // Server receive auth switch response packet from client. + ByteBuffer authSwitchResponse = channel.fetchOnePacket(); + int length = authSwitchResponse.limit(); + password = new byte[length]; + System.arraycopy(authSwitchResponse.array(), 0, password, 0, length); + } } // For safety, not allowed to change to root or admin. diff --git a/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlCapabilityTest.java b/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlCapabilityTest.java index d0755742fd6..ed2f7b97506 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlCapabilityTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlCapabilityTest.java @@ -44,12 +44,30 @@ public class MysqlCapabilityTest { public void testDefaultFlags() { MysqlCapability capability = MysqlCapability.DEFAULT_CAPABILITY; Assert.assertEquals("CLIENT_LONG_FLAG | CLIENT_CONNECT_WITH_DB | CLIENT_LOCAL_FILES | CLIENT_PROTOCOL_41" - + " | CLIENT_SECURE_CONNECTION | CLIENT_PLUGIN_AUTH | CLIENT_CONNECT_ATTRS" + + " | CLIENT_SECURE_CONNECTION | CLIENT_MULTI_STATEMENTS | CLIENT_MULTI_RESULTS" + + " | CLIENT_PS_MULTI_RESULTS | CLIENT_PLUGIN_AUTH | CLIENT_CONNECT_ATTRS" + " | CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA | CLIENT_DEPRECATE_EOF", capability.toString()); Assert.assertTrue(capability.supportClientLocalFile()); Assert.assertTrue(capability.isConnectAttrs()); Assert.assertTrue(capability.isPluginAuthDataLengthEncoded()); Assert.assertTrue(capability.isDeprecatedEOF()); + Assert.assertTrue(capability.isClientMultiStatements()); + Assert.assertFalse(capability.isClientUseSsl()); + } + + @Test + public void testSslFlags() { + MysqlCapability capability = MysqlCapability.SSL_CAPABILITY; + Assert.assertEquals( + MysqlCapability.DEFAULT_CAPABILITY.getFlags() | MysqlCapability.Flag.CLIENT_SSL.getFlagBit(), + capability.getFlags()); + Assert.assertEquals("CLIENT_LONG_FLAG | CLIENT_CONNECT_WITH_DB | CLIENT_LOCAL_FILES | CLIENT_PROTOCOL_41" + + " | CLIENT_SSL | CLIENT_SECURE_CONNECTION | CLIENT_MULTI_STATEMENTS | CLIENT_MULTI_RESULTS" + + " | CLIENT_PS_MULTI_RESULTS | CLIENT_PLUGIN_AUTH | CLIENT_CONNECT_ATTRS" + + " | CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA | CLIENT_DEPRECATE_EOF", + capability.toString()); + Assert.assertTrue(capability.isClientUseSsl()); + Assert.assertTrue(capability.isClientMultiStatements()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlHandshakePacketTest.java b/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlHandshakePacketTest.java index 2434842dcf4..95d95d8eb7f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlHandshakePacketTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlHandshakePacketTest.java @@ -27,6 +27,9 @@ import org.junit.Test; import java.nio.ByteBuffer; public class MysqlHandshakePacketTest { + private static final int MULTI_RESULT_FLAGS = MysqlCapability.Flag.CLIENT_MULTI_RESULTS.getFlagBit() + | MysqlCapability.Flag.CLIENT_PS_MULTI_RESULTS.getFlagBit(); + private byte[] buf; private MysqlCapability capability; @@ -81,6 +84,11 @@ public class MysqlHandshakePacketTest { MysqlCapability advertisedCapability = new MysqlCapability(flags); Assert.assertTrue(advertisedCapability.isConnectAttrs()); Assert.assertTrue(advertisedCapability.isPluginAuthDataLengthEncoded()); + // The multi-statement flags MySQL advertises, in the upper two bytes: a driver that masks + // what it asks for with the advertised set (Connector/J 8 and 9) keeps CLIENT_MULTI_STATEMENTS + // only when they are there. + Assert.assertTrue(advertisedCapability.isClientMultiStatements()); + Assert.assertEquals(MULTI_RESULT_FLAGS, flags & MULTI_RESULT_FLAGS); // length of plugin data Assert.assertEquals(21, MysqlProto.readInt1(buffer)); // length of plugin data @@ -99,5 +107,4 @@ public class MysqlHandshakePacketTest { Assert.assertEquals(0, MysqlProto.readInt1(buffer)); Assert.assertEquals(22, buffer.remaining()); } - } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlProtoTest.java b/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlProtoTest.java index 607f5e39248..89c879d7bf5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlProtoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlProtoTest.java @@ -289,6 +289,47 @@ public class MysqlProtoTest { Assert.assertTrue(MysqlProto.negotiate(context)); } + @Test + public void testNegotiateSetsClientMultiStatementsWhenTheClientAsksForIt() throws Exception { + // The client echoes the advertised capabilities, which carry CLIENT_MULTI_STATEMENTS: what + // Connector/J 8 and 9 with allowMultiQueries send, since they mask the flags they ask for + // with the advertised ones. + mockChannel("user", true); + mockPassword(true); + mockAccess(); + ConnectContext context = new ConnectContext(streamConnection); + context.setEnv(env); + context.setThreadLocalInfo(); + Assert.assertTrue(MysqlProto.negotiate(context)); + Assert.assertTrue(context.getCapability().isClientMultiStatements()); + new Verifications() { + { + channel.setClientMultiStatements(); + times = 1; + } + }; + } + + @Test + public void testNegotiateLeavesClientMultiStatementsWhenTheClientDoesNotAskForIt() throws Exception { + int clientFlags = MysqlCapability.DEFAULT_CAPABILITY.getFlags() + & ~MysqlCapability.Flag.CLIENT_MULTI_STATEMENTS.getFlagBit(); + mockChannel("user", true, clientFlags); + mockPassword(true); + mockAccess(); + ConnectContext context = new ConnectContext(streamConnection); + context.setEnv(env); + context.setThreadLocalInfo(); + Assert.assertTrue(MysqlProto.negotiate(context)); + Assert.assertFalse(context.getCapability().isClientMultiStatements()); + new Verifications() { + { + channel.setClientMultiStatements(); + times = 0; + } + }; + } + @Test public void testNegotiateUsesClientServerCapabilityIntersection() throws Exception { int clientFlags = MysqlCapability.DEFAULT_CAPABILITY.getFlags() diff --git a/fe/fe-core/src/test/java/org/apache/doris/mysql/authenticate/password/NativePasswordResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/mysql/authenticate/password/NativePasswordResolverTest.java new file mode 100644 index 00000000000..c6c7268a74e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mysql/authenticate/password/NativePasswordResolverTest.java @@ -0,0 +1,233 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mysql.authenticate.password; + +import org.apache.doris.common.Config; +import org.apache.doris.mysql.CachingSha2PasswordExchange; +import org.apache.doris.mysql.MysqlAuthPacket; +import org.apache.doris.mysql.MysqlCapability; +import org.apache.doris.mysql.MysqlChannel; +import org.apache.doris.mysql.MysqlHandshakePacket; +import org.apache.doris.mysql.MysqlPassword; +import org.apache.doris.mysql.MysqlSerializer; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.QueryState; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.KeyFactory; +import java.security.PublicKey; +import java.security.spec.X509EncodedKeySpec; +import java.util.Base64; +import java.util.Map; +import javax.crypto.Cipher; + +class NativePasswordResolverTest { + private static final Map<String, String> MYSQL_9_CLIENT = ImmutableMap.of("_client_name", "libmysql", + CachingSha2PasswordExchange.CLIENT_VERSION_ATTR, "9.4.0"); + private static final Map<String, String> MYSQL_8_CLIENT = ImmutableMap.of("_client_name", "libmysql", + CachingSha2PasswordExchange.CLIENT_VERSION_ATTR, "8.0.44"); + + private String previousMode; + private final MysqlHandshakePacket handshake = new MysqlHandshakePacket(1); + private final MysqlChannel channel = Mockito.mock(MysqlChannel.class); + + @BeforeEach + void setUp() { + previousMode = Config.mysql_caching_sha2_password_clients; + Mockito.when(channel.getRemoteIp()).thenReturn("127.0.0.1"); + } + + @AfterEach + void tearDown() { + Config.mysql_caching_sha2_password_clients = previousMode; + } + + @Test + void testMysql9ClientGetsFullAuthenticationOverRsa() throws Exception { + byte[] nonce = handshake.getAuthPluginData(); + Mockito.when(channel.fetchOnePacket()).thenReturn( + ByteBuffer.wrap(new byte[32]), // its caching_sha2 scramble, ignored + ByteBuffer.wrap(new byte[] {0x02}), // asks for the public key + ByteBuffer.wrap(encryptLikeClient("secret", nonce, "RSA/ECB/OAEPWithSHA-1AndMGF1Padding"))); + + NativePassword password = resolve(authPacket(MysqlCapability.DEFAULT_CAPABILITY, MYSQL_9_CLIENT)); + + Assertions.assertArrayEquals(MysqlPassword.scramble(nonce, "secret"), password.getRemotePasswd()); + Assertions.assertArrayEquals(nonce, password.getRandomString()); + // AuthSwitchRequest, perform-full-authentication, public key + Mockito.verify(channel, Mockito.times(3)).sendAndFlush(Mockito.any()); + } + + @Test + void testConnectorJPkcs1PaddingIsAcceptedToo() throws Exception { + // Connector/J takes the padding from the server version and Doris reports 5.7: PKCS#1 v1.5 + byte[] nonce = handshake.getAuthPluginData(); + Mockito.when(channel.fetchOnePacket()).thenReturn( + ByteBuffer.wrap(new byte[32]), + ByteBuffer.wrap(new byte[] {0x02}), + ByteBuffer.wrap(encryptLikeClient("secret", nonce, "RSA/ECB/PKCS1Padding"))); + + NativePassword password = resolve(authPacket(MysqlCapability.DEFAULT_CAPABILITY, MYSQL_9_CLIENT)); + + Assertions.assertArrayEquals(MysqlPassword.scramble(nonce, "secret"), password.getRemotePasswd()); + } + + @Test + void testUndecryptableAnswerIsRejectedWithAnError() throws Exception { + Mockito.when(channel.fetchOnePacket()).thenReturn( + ByteBuffer.wrap(new byte[32]), + ByteBuffer.wrap(new byte[] {0x02}), + ByteBuffer.wrap(new byte[256])); + Mockito.when(channel.getSerializer()).thenReturn(MysqlSerializer.newInstance()); + ConnectContext context = Mockito.mock(ConnectContext.class); + QueryState state = new QueryState(); + Mockito.when(context.getState()).thenReturn(state); + Mockito.when(context.getMysqlChannel()).thenReturn(channel); + + Assertions.assertFalse(new NativePasswordResolver().resolvePassword(context, channel, + MysqlSerializer.newInstance(), authPacket(MysqlCapability.DEFAULT_CAPABILITY, MYSQL_9_CLIENT), + handshake).isPresent()); + Assertions.assertEquals(QueryState.MysqlStateType.ERR, state.getStateType()); + // AuthSwitchRequest, perform-full-authentication, public key, the error packet + Mockito.verify(channel, Mockito.times(4)).sendAndFlush(Mockito.any()); + } + + @Test + void testMysql9ClientSendsThePlaintextOverTls() throws Exception { + byte[] nonce = handshake.getAuthPluginData(); + Mockito.when(channel.fetchOnePacket()).thenReturn( + ByteBuffer.wrap(new byte[32]), + ByteBuffer.wrap("secret\0".getBytes(StandardCharsets.UTF_8))); + + // The negotiated capabilities omit CLIENT_SSL even after TLS has been established. + Mockito.when(channel.isSslMode()).thenReturn(true); + NativePassword password = resolve(authPacket(MysqlCapability.DEFAULT_CAPABILITY, MYSQL_9_CLIENT)); + + Assertions.assertArrayEquals(MysqlPassword.scramble(nonce, "secret"), password.getRemotePasswd()); + Mockito.verify(channel, Mockito.times(2)).sendAndFlush(Mockito.any()); + } + + @Test + void testEmptyPasswordNeedsNoFullAuthentication() throws Exception { + // libmysqlclient answers the switch with the empty string and its terminator: one NUL byte + Mockito.when(channel.fetchOnePacket()).thenReturn(ByteBuffer.wrap(new byte[] {0})); + + NativePassword password = resolve(authPacket(MysqlCapability.DEFAULT_CAPABILITY, MYSQL_9_CLIENT)); + + Assertions.assertEquals(0, password.getRemotePasswd().length); + Mockito.verify(channel, Mockito.times(1)).sendAndFlush(Mockito.any()); + } + + @Test + void testMysql8ClientIsStillSwitchedToNativePassword() throws Exception { + byte[] nativeResponse = new byte[20]; + nativeResponse[0] = 7; + Mockito.when(channel.fetchOnePacket()).thenReturn(ByteBuffer.wrap(nativeResponse)); + + NativePassword password = resolve(authPacket(MysqlCapability.DEFAULT_CAPABILITY, MYSQL_8_CLIENT)); + + Assertions.assertArrayEquals(nativeResponse, password.getRemotePasswd()); + Mockito.verify(channel, Mockito.times(1)).sendAndFlush(Mockito.any()); + } + + @Test + void testOversizedClientVersionStillUsesFullAuthentication() throws Exception { + Config.mysql_caching_sha2_password_clients = "auto"; + Map<String, String> attributes = ImmutableMap.of(CachingSha2PasswordExchange.CLIENT_VERSION_ATTR, + "2147483648.0"); + byte[] nonce = handshake.getAuthPluginData(); + Mockito.when(channel.fetchOnePacket()).thenReturn( + ByteBuffer.wrap(new byte[32]), + ByteBuffer.wrap("secret\0".getBytes(StandardCharsets.UTF_8))); + Mockito.when(channel.isSslMode()).thenReturn(true); + + Assertions.assertTrue(CachingSha2PasswordExchange.serves(CachingSha2PasswordExchange.PLUGIN_NAME, + attributes)); + NativePassword password = resolve(authPacket(MysqlCapability.DEFAULT_CAPABILITY, attributes)); + + Assertions.assertArrayEquals(MysqlPassword.scramble(nonce, "secret"), password.getRemotePasswd()); + Mockito.verify(channel, Mockito.times(2)).sendAndFlush(Mockito.any()); + } + + @Test + void testModeNoneSwitchesEveryClientAndModeAllServesEveryClient() throws Exception { + Config.mysql_caching_sha2_password_clients = "none"; + Assertions.assertFalse(CachingSha2PasswordExchange.serves(CachingSha2PasswordExchange.PLUGIN_NAME, + MYSQL_9_CLIENT)); + Config.mysql_caching_sha2_password_clients = "all"; + Assertions.assertTrue(CachingSha2PasswordExchange.serves(CachingSha2PasswordExchange.PLUGIN_NAME, + ImmutableMap.of())); + Config.mysql_caching_sha2_password_clients = "auto"; + Assertions.assertFalse(CachingSha2PasswordExchange.serves(CachingSha2PasswordExchange.PLUGIN_NAME, + ImmutableMap.of())); + Assertions.assertFalse(CachingSha2PasswordExchange.serves(CachingSha2PasswordExchange.PLUGIN_NAME, + ImmutableMap.of(CachingSha2PasswordExchange.CLIENT_VERSION_ATTR, "00000000000000000008.0"))); + Assertions.assertTrue(CachingSha2PasswordExchange.serves(CachingSha2PasswordExchange.PLUGIN_NAME, + ImmutableMap.of(CachingSha2PasswordExchange.CLIENT_VERSION_ATTR, "00000000000000000009.0"))); + Assertions.assertFalse(CachingSha2PasswordExchange.serves("mysql_native_password", MYSQL_9_CLIENT)); + Assertions.assertFalse(CachingSha2PasswordExchange.serves(CachingSha2PasswordExchange.PLUGIN_NAME, + ImmutableMap.of(CachingSha2PasswordExchange.CLIENT_VERSION_ATTR, "x"))); + } + + @Test + void testClientClosingTheConnectionMidExchangeYieldsNoRequest() throws Exception { + Mockito.when(channel.fetchOnePacket()).thenReturn(ByteBuffer.wrap(new byte[32]), (ByteBuffer) null); + + Assertions.assertFalse(new NativePasswordResolver().resolvePassword(Mockito.mock(ConnectContext.class), channel, + MysqlSerializer.newInstance(), authPacket(MysqlCapability.DEFAULT_CAPABILITY, MYSQL_9_CLIENT), + handshake).isPresent()); + } + + private NativePassword resolve(MysqlAuthPacket authPacket) throws Exception { + return (NativePassword) new NativePasswordResolver().resolvePassword(Mockito.mock(ConnectContext.class), + channel, MysqlSerializer.newInstance(), authPacket, handshake) + .orElseThrow(() -> new AssertionError("password is required")); + } + + private static MysqlAuthPacket authPacket(MysqlCapability capability, Map<String, String> attributes) { + MysqlAuthPacket authPacket = Mockito.mock(MysqlAuthPacket.class); + Mockito.when(authPacket.getCapability()).thenReturn(capability); + Mockito.when(authPacket.getPluginName()).thenReturn(CachingSha2PasswordExchange.PLUGIN_NAME); + Mockito.when(authPacket.getConnectAttributes()).thenReturn(attributes); + Mockito.when(authPacket.getAuthResponse()).thenReturn(new byte[0]); + return authPacket; + } + + // What a client does with the PEM key the server sent: RSA over (password + NUL) XOR nonce + private static byte[] encryptLikeClient(String password, byte[] nonce, String transformation) throws Exception { + String pem = CachingSha2PasswordExchange.publicKeyPem() + .replace("-----BEGIN PUBLIC KEY-----", "").replace("-----END PUBLIC KEY-----", "").replace("\n", ""); + PublicKey key = KeyFactory.getInstance("RSA") + .generatePublic(new X509EncodedKeySpec(Base64.getDecoder().decode(pem))); + byte[] plain = (password + "\0").getBytes(StandardCharsets.UTF_8); + for (int i = 0; i < plain.length; i++) { + plain[i] ^= nonce[i % nonce.length]; + } + Cipher cipher = Cipher.getInstance(transformation); + cipher.init(Cipher.ENCRYPT_MODE, key); + return cipher.doFinal(plain); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2LiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2LiteralTest.java index 96775d44bfc..a05786857da 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2LiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2LiteralTest.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.expressions.literal; +import org.apache.doris.catalog.MysqlColType; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.types.DateTimeV2Type; @@ -26,6 +27,9 @@ import org.apache.doris.nereids.types.TimeV2Type; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + public class TimeV2LiteralTest { @Test @@ -186,4 +190,110 @@ public class TimeV2LiteralTest { Assertions.assertEquals(59, dateTime.getSecond()); } + @Test + public void testFromMysqlBinaryTimeParameter() throws AnalysisException { + // Protocol::MYSQL_TYPE_TIME, 12 bytes: is_negative, days, hours, minutes, seconds, microseconds + ByteBuffer data = ByteBuffer.allocate(13).order(ByteOrder.LITTLE_ENDIAN); + data.put((byte) 12).put((byte) 0).putInt(0).put((byte) 12).put((byte) 34).put((byte) 56).putInt(123456).flip(); + Literal literal = Literal.getLiteralByMysqlType(MysqlColType.MYSQL_TYPE_TIME, false, data); + TimeV2Literal time = Assertions.assertInstanceOf(TimeV2Literal.class, literal); + Assertions.assertEquals("12:34:56.123456", time.getStringValue()); + + // 8 bytes: no microseconds; negative, and the days fold into the hours + data = ByteBuffer.allocate(9).order(ByteOrder.LITTLE_ENDIAN); + data.put((byte) 8).put((byte) 1).putInt(1).put((byte) 1).put((byte) 2).put((byte) 3).flip(); + time = (TimeV2Literal) Literal.getLiteralByMysqlType(MysqlColType.MYSQL_TYPE_TIME2, false, data); + Assertions.assertEquals("-25:02:03.000000", time.getStringValue()); + + // 0 bytes: 00:00:00 + data = ByteBuffer.allocate(1).order(ByteOrder.LITTLE_ENDIAN); + data.put((byte) 0).flip(); + time = (TimeV2Literal) Literal.getLiteralByMysqlType(MysqlColType.MYSQL_TYPE_TIME, false, data); + Assertions.assertEquals("00:00:00.000000", time.getStringValue()); + } + + @Test + public void testBinaryTimeRejectsInvalidFraming() { + for (MysqlColType type : new MysqlColType[] {MysqlColType.MYSQL_TYPE_TIME, MysqlColType.MYSQL_TYPE_TIME2}) { + Assertions.assertThrows(AnalysisException.class, + () -> Literal.getLiteralByMysqlType(type, false, ByteBuffer.allocate(0))); + for (int len : new int[] {1, 7, 9, 11, 13, 252, 253, 254, 255}) { + ByteBuffer data = ByteBuffer.allocate(17).order(ByteOrder.LITTLE_ENDIAN); + data.put((byte) len).position(17); + data.flip(); + Assertions.assertThrows(AnalysisException.class, + () -> Literal.getLiteralByMysqlType(type, false, data), "length=" + len); + Assertions.assertEquals(1, data.position(), "invalid length must not consume parameter data"); + } + for (int len : new int[] {8, 12}) { + for (int available = 0; available < len; available++) { + ByteBuffer data = ByteBuffer.allocate(available + 1).order(ByteOrder.LITTLE_ENDIAN); + data.put((byte) len).position(available + 1); + data.flip(); + Assertions.assertThrows(AnalysisException.class, + () -> Literal.getLiteralByMysqlType(type, false, data)); + } + } + } + } + + @Test + public void testBinaryTimeKeepsParameterBoundaries() { + // The malformed first value used to read the second TIME's length as days=8. + ByteBuffer malformed = ByteBuffer.wrap(new byte[] {1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0}) + .order(ByteOrder.LITTLE_ENDIAN); + Assertions.assertThrows(AnalysisException.class, + () -> Literal.getLiteralByMysqlType(MysqlColType.MYSQL_TYPE_TIME, false, malformed)); + Assertions.assertEquals(1, malformed.position()); + + ByteBuffer valid = ByteBuffer.allocate(13).order(ByteOrder.LITTLE_ENDIAN); + valid.put((byte) 8).put((byte) 0).putInt(0).put((byte) 12).put((byte) 34).put((byte) 56) + .putInt(7654321).flip(); + Assertions.assertEquals("12:34:56.000000", + Literal.getLiteralByMysqlType(MysqlColType.MYSQL_TYPE_TIME, false, valid).getStringValue()); + Assertions.assertEquals(7654321, + Literal.getLiteralByMysqlType(MysqlColType.MYSQL_TYPE_LONG, false, valid).getValue()); + Assertions.assertFalse(valid.hasRemaining()); + } + + @Test + public void testBinaryTimeRejectsOutOfRangeDurations() { + for (MysqlColType type : new MysqlColType[] {MysqlColType.MYSQL_TYPE_TIME, MysqlColType.MYSQL_TYPE_TIME2}) { + for (int days : new int[] {35, 0x20000000, 0x80000000, 0xffffffff}) { + ByteBuffer data = binaryTime(0, days, 0, 0, 0, 0); + Assertions.assertThrows(AnalysisException.class, + () -> Literal.getLiteralByMysqlType(type, false, data)); + } + Assertions.assertThrows(AnalysisException.class, + () -> Literal.getLiteralByMysqlType(type, false, binaryTime(0, 34, 23, 0, 0, 0))); + Assertions.assertThrows(AnalysisException.class, + () -> Literal.getLiteralByMysqlType(type, false, binaryTime(0, 0, 0, 0, 0, 1000000))); + for (int sign : new int[] {2, 255}) { + Assertions.assertThrows(AnalysisException.class, + () -> Literal.getLiteralByMysqlType(type, false, binaryTime(sign, 0, 0, 0, 0, 0))); + } + } + } + + @Test + public void testBinaryTimeAcceptsMaximumDuration() { + for (MysqlColType type : new MysqlColType[] {MysqlColType.MYSQL_TYPE_TIME, MysqlColType.MYSQL_TYPE_TIME2}) { + Assertions.assertEquals("838:59:59.999999", + Literal.getLiteralByMysqlType(type, false, binaryTime(0, 34, 22, 59, 59, 999999)).getStringValue()); + Assertions.assertEquals("-838:59:59.999999", + Literal.getLiteralByMysqlType(type, false, binaryTime(1, 34, 22, 59, 59, 999999)).getStringValue()); + } + } + + private static ByteBuffer binaryTime(int sign, int days, int hour, int minute, int second, int microsecond) { + int len = microsecond == 0 ? 8 : 12; + ByteBuffer data = ByteBuffer.allocate(len + 1).order(ByteOrder.LITTLE_ENDIAN); + data.put((byte) len).put((byte) sign).putInt(days).put((byte) hour).put((byte) minute).put((byte) second); + if (len == 12) { + data.putInt(microsecond); + } + data.flip(); + return data; + } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowConfigCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowConfigCommandTest.java index 714a531aa26..245a3353606 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowConfigCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowConfigCommandTest.java @@ -124,9 +124,8 @@ public class ShowConfigCommandTest extends TestWithFeService { public void testMaskAllPasswordConfigsWithPattern() throws Exception { ShowResultSet resultSet = runShowFrontendConfig("%password%"); List<List<String>> rows = resultSet.getResultRows(); - // The pattern '%password%' matches 6 configs: the 5 sensitive ones plus - // tls_cert_based_auth_ignore_password which is not a secret. - Assertions.assertEquals(SENSITIVE_KEYS.size() + 1, rows.size()); + // The pattern '%password%' matches the sensitive configs and two non-sensitive configs. + Assertions.assertEquals(SENSITIVE_KEYS.size() + 2, rows.size()); Map<String, String> keyToValue = rows.stream() .collect(Collectors.toMap(row -> row.get(0), row -> row.get(1))); for (String sensitiveKey : SENSITIVE_KEYS) { @@ -136,6 +135,8 @@ public class ShowConfigCommandTest extends TestWithFeService { } // A non-sensitive config containing "password" in its name keeps its real value. Assertions.assertEquals("false", keyToValue.get("tls_cert_based_auth_ignore_password")); + Assertions.assertEquals(Config.mysql_caching_sha2_password_clients, + keyToValue.get("mysql_caching_sha2_password_clients")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java index c7239481860..1bdd89b5e4f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java @@ -43,6 +43,7 @@ import org.junit.After; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.jupiter.api.Assertions; import java.io.File; import java.util.UUID; @@ -111,6 +112,17 @@ public class VariableMgrTest { Assert.fail("No exception throws."); } + @Test + public void testAuthenticationPolicyIsTheReadOnlyMysqlDefault() throws Exception { + SessionVariable sessionVariable = new SessionVariable(); + VariableExpr variableExpr = new VariableExpr(GlobalVariable.AUTHENTICATION_POLICY); + VariableMgr.fillValue(sessionVariable, variableExpr); + Assertions.assertEquals("*,,", VariableMgr.getValue(sessionVariable, variableExpr)); + + SetVar setVar = new SetVar(SetType.GLOBAL, GlobalVariable.AUTHENTICATION_POLICY, new StringLiteral("*,")); + Assertions.assertThrows(DdlException.class, () -> VariableMgr.setVar(sessionVariable, setVar)); + } + @Test public void testVariableCallback() throws Exception { SetOptionsCommand stmt = (SetOptionsCommand) UtFrameUtils.parseStmt( --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
