jamesfredley commented on code in PR #16631: URL: https://github.com/apache/grails-core/pull/16631#discussion_r4352333988
########## grails-datamapping-core/src/main/groovy/org/grails/datastore/gorm/jdbc/sqltail/SqlTailRedactor.java: ########## @@ -0,0 +1,360 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.jdbc.sqltail; + +import java.util.ArrayList; +import java.util.List; + +/** + * Takes the values out of a statement before it is captured. Every string, number and + * dollar-quoted literal becomes {@link #PLACEHOLDER} and every comment is dropped, so that a value + * written into the SQL text, a password in a native query or a key in a comment, never reaches the + * capture file. Only the recorded copy is redacted: the statement the database runs is not changed. + * + * <p>Databases disagree on parts of the lexical syntax: whether a backslash escapes a quote, whether + * comments nest, whether double quotes delimit an identifier or a string, and whether {@code #} + * starts a comment. A statement is redacted under every reading its {@link Syntax} allows, and is + * recorded only when all of them give the same text. When they do not, or when a literal or a + * comment never ends, the statement is withheld: {@link #WITHHELD} is recorded in its place, never + * the statement.</p> + * + * @since 8.1 + */ +public final class SqlTailRedactor { + + /** What a literal is replaced with. */ + public static final String PLACEHOLDER = "?"; + + /** What is recorded in place of a statement whose values cannot be told apart from its SQL. */ + public static final String WITHHELD = "(statement withheld: its values could not be redacted)"; + + private static final boolean[] NO = {false}; + private static final boolean[] EITHER = {false, true}; + + /** + * The lexical syntax of the database a statement runs on, as far as finding its values goes. + */ + public enum Syntax { + + /** + * Not known, or a database that can read double-quoted text as a string although it quotes + * identifiers with double quotes, such as SQLite and Informix: a statement is recorded only when + * the readings of every syntax below agree. + */ + UNKNOWN, + + /** Standard SQL, where double quotes delimit identifiers: H2, PostgreSQL, Oracle, DB2, HSQLDB and HANA. */ + STANDARD, + + /** MySQL and MariaDB: double quotes delimit strings, and {@code #} and {@code -- } start comments. */ + MYSQL, + + /** + * Microsoft SQL Server and Sybase: square brackets delimit identifiers, comments nest, and double + * quotes delimit a string when {@code QUOTED_IDENTIFIER} is off, as it is by default on Sybase. + */ + TRANSACT_SQL + } + + private SqlTailRedactor() { + } + + /** + * Redacts a statement. + * + * @param sql the statement + * @param syntax the syntax of the database it runs on + * @return the statement with its literals replaced and its comments dropped, or {@link #WITHHELD} + */ + public static String redact(String sql, Syntax syntax) { + if (sql == null) { + return null; + } + String redacted = null; + for (Reading reading : readings(sql, syntax == null ? Syntax.UNKNOWN : syntax)) { + String candidate = reading.redact(sql); + if (candidate == null || (redacted != null && !redacted.equals(candidate))) { + return WITHHELD; + } + redacted = candidate; + } + return redacted; + } + + /** + * The readings a statement has to agree under. A reading that depends on a character the + * statement does not contain is the same as the one that does not, so it is left out. + */ + private static List<Reading> readings(String sql, Syntax syntax) { + boolean[] backslash = sql.indexOf('\\') >= 0 ? EITHER : NO; + int comment = sql.indexOf("/*"); + boolean[] nesting = comment >= 0 && sql.indexOf("/*", comment + 2) >= 0 ? EITHER : NO; + List<Reading> readings = new ArrayList<>(); + if (syntax == Syntax.STANDARD || syntax == Syntax.UNKNOWN) { + for (boolean backslashEscapes : backslash) { + for (boolean nestedComments : nesting) { + readings.add(new Reading(Syntax.STANDARD, backslashEscapes, nestedComments)); + } + } + } + if (syntax == Syntax.MYSQL || syntax == Syntax.UNKNOWN) { + for (boolean backslashEscapes : backslash) { + readings.add(new Reading(Syntax.MYSQL, backslashEscapes, false)); + } + } + if (syntax == Syntax.TRANSACT_SQL || syntax == Syntax.UNKNOWN) { + readings.add(new Reading(Syntax.TRANSACT_SQL, false, true)); + } + return readings; + } + + /** + * One way of reading a statement. Only MySQL delimits identifiers with backticks, starts a comment + * with {@code #}, and needs whitespace after {@code --}; only Transact-SQL delimits identifiers with + * square brackets. Outside standard SQL, double quotes can delimit a string: in MySQL unless + * {@code ANSI_QUOTES} is set, and in Transact-SQL when {@code QUOTED_IDENTIFIER} is off. Either way the + * quoted run ends at the same place, so it is replaced as a value. + * + * @param syntax the syntax read in, never {@link Syntax#UNKNOWN} + * @param backslashEscapes whether a backslash escapes the character after it in a string + * @param nestedComments whether a block comment can contain another + */ + private record Reading(Syntax syntax, boolean backslashEscapes, boolean nestedComments) { + + /** + * @return the redacted statement, or null when a literal, a quoted identifier or a comment never ends + */ + String redact(String sql) { + boolean mysql = syntax == Syntax.MYSQL; + boolean doubleQuotedValue = syntax != Syntax.STANDARD; + int length = sql.length(); + StringBuilder out = new StringBuilder(length); + int i = 0; + while (i < length) { + char c = sql.charAt(i); + char next = i + 1 < length ? sql.charAt(i + 1) : 0; + char previous = i > 0 ? sql.charAt(i - 1) : 0; + int dollarTag = c == '$' && !isIdentifierPart(previous) ? dollarTagLength(sql, i) : 0; + int end; + if (c == '\'') { + end = endOfQuotedLiteral(sql, i); + out.append(PLACEHOLDER); + } + else if (c == '"') { Review Comment: Spanner is classified as MySQL, and GoogleSQL triple quotes are not a literal. `select """a"example-secret"b"""` becomes `select ?example-secret?`, so part of the value is written. Handle the triple-quoted form, or withhold a statement that contains it. ########## grails-datamapping-core/src/cli/groovy/org/grails/datastore/gorm/jdbc/sqltail/cli/SqlTailFileReader.groovy: ########## @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.jdbc.sqltail.cli + +import java.nio.charset.StandardCharsets + +import groovy.transform.CompileStatic + +import org.grails.datastore.gorm.jdbc.sqltail.SqlTailRecord + +/** + * Follows a capture file the way {@code tail -f} does: each poll returns the records written + * since the last one. A line that has no terminator yet is left for the next poll, so a record + * the application is still writing is never read in half. An application that starts again + * truncates the file and names its new run in the first line; when that line names another run, + * or the file shrinks, reading starts over from the beginning. The run tells a restart apart from + * a file that grew past where the reader was before it was next polled. + * + * @since 8.1 + */ +@CompileStatic +class SqlTailFileReader implements Closeable { + + private static final byte NEWLINE = (byte) 10 + + /** How much of the file the first line is looked for in; a header is far shorter. */ + private static final int HEADER_LIMIT = 256 + + final File file + private long position + private String run + private boolean restarted + + /** + * @param file the capture file + * @param fromStart whether to read what the file already holds, or only what is written from now on + */ + SqlTailFileReader(File file, boolean fromStart) { + this.file = file + this.position = fromStart ? 0 : file.length() + this.run = file.exists() ? readRun() : null + } + + /** + * @return the records written since the last poll, or since the reader was opened + */ + List<SqlTailRecord> poll() { + restarted = false + long length = file.length() + String current = length > 0 ? readRun() : null + if (length < position || (current != null && run != null && current != run)) { + position = 0 + restarted = true + } + if (current != null || restarted) { + // a file truncated before its header was written names its run on a later poll + run = current + } + if (length <= position) { + return [] + } + byte[] bytes = readFrom(position, length) + int consumed = lastLineEnd(bytes) + if (consumed == 0) { + return [] + } + position += consumed + String text = new String(bytes, 0, consumed, StandardCharsets.UTF_8) + List<SqlTailRecord> records = [] + for (String line : text.split('\n')) { + SqlTailRecord record = SqlTailRecord.parse(line) + if (record != null) { + records.add(record) + } + } + records + } + + /** + * @return whether the last poll found a new run in the file, which an application restart writes + */ + boolean isRestarted() { + restarted + } + + /** + * @return how far the file has been read + */ + long getPosition() { + position + } + + /** + * @return the run that wrote the file, as its first line names it; null before the line is complete + */ + String getRun() { + run + } + + @Override + void close() { + // the file is opened and closed per poll + } + + private static int lastLineEnd(byte[] bytes) { + for (int i = bytes.length - 1; i >= 0; i--) { + if (bytes[i] == NEWLINE) { + return i + 1 + } + } + 0 + } + + private String readRun() { + byte[] head = readFrom(0, Math.min(file.length(), (long) HEADER_LIMIT)) + for (int i = 0; i < head.length; i++) { + if (head[i] == NEWLINE) { + return SqlTailRecord.parseHeader(new String(head, 0, i, StandardCharsets.UTF_8)) + } + } + null + } + + private byte[] readFrom(long from, long to) { + RandomAccessFile input = new RandomAccessFile(file, 'r') + try { + input.seek(from) + byte[] bytes = new byte[(int) Math.max(0L, Math.min(to, input.length()) - from)] Review Comment: If the application truncates the capture between `length()` and `readFully`, the read throws `EOFException` and sql-tail exits. A restart should be a retry, not a command failure. Treat a short read caused by truncation as a restart and poll again. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
