jdaugherty commented on code in PR #16631: URL: https://github.com/apache/grails-core/pull/16631#discussion_r4360312343
########## grails-datamapping-core/src/main/groovy/org/grails/datastore/gorm/jdbc/sqltail/SqlTailRedactor.java: ########## @@ -0,0 +1,414 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.jdbc.sqltail; + +import java.util.ArrayList; +import java.util.List; + +/** + * Takes the values out of a statement before it is captured. Every string, number and + * dollar-quoted literal becomes {@link #PLACEHOLDER} and every comment is dropped, so that a value + * written into the SQL text, a password in a native query or a key in a comment, never reaches the + * capture file. Only the recorded copy is redacted: the statement the database runs is not changed. + * + * <p>Databases disagree on parts of the lexical syntax: whether a backslash escapes a quote, whether + * comments nest, whether double quotes delimit an identifier or a string, whether a quote written + * twice escapes it or ends one string and starts another, whether a quote written three times + * opens a string that quotes can appear in, and whether {@code #} starts a comment. A statement is + * redacted under every reading its {@link Syntax} allows, and is recorded only when all of them give + * the same text. When they do not, or when a literal or a comment never ends, the statement is + * withheld: {@link #WITHHELD} is recorded in its place, never the statement.</p> + * + * @since 8.1 + */ +public final class SqlTailRedactor { + + /** What a literal is replaced with. */ + public static final String PLACEHOLDER = "?"; + + /** What is recorded in place of a statement whose values cannot be told apart from its SQL. */ + public static final String WITHHELD = "(statement withheld: its values could not be redacted)"; + + private static final boolean[] NO = {false}; + private static final boolean[] EITHER = {false, true}; + + /** + * The lexical syntax of the database a statement runs on, as far as finding its values goes. + */ + public enum Syntax { + + /** + * Not known, or a database that can read double-quoted text as a string although it quotes + * identifiers with double quotes, such as SQLite and Informix: a statement is recorded only when + * the readings of every syntax below agree. + */ + UNKNOWN, + + /** Standard SQL, where double quotes delimit identifiers: H2, PostgreSQL, Oracle, DB2, HSQLDB and HANA. */ + STANDARD, + + /** MySQL and MariaDB: double quotes delimit strings, and {@code #} and {@code -- } start comments. */ + MYSQL, + + /** + * Microsoft SQL Server and Sybase: square brackets delimit identifiers, comments nest, and double + * quotes delimit a string when {@code QUOTED_IDENTIFIER} is off, as it is by default on Sybase. + */ + TRANSACT_SQL, + + /** + * GoogleSQL, the dialect of Cloud Spanner: backticks delimit identifiers, single and double quotes + * delimit strings, a string that opens with its quote written three times ends only at the quote + * written three times again and holds any other quote, a backslash always escapes the character + * after it, a quote written twice is two strings, and {@code #} starts a comment. + */ + GOOGLE_SQL + } + + private SqlTailRedactor() { + } + + /** + * Redacts a statement. + * + * @param sql the statement + * @param syntax the syntax of the database it runs on + * @return the statement with its literals replaced and its comments dropped, or {@link #WITHHELD} + */ + public static String redact(String sql, Syntax syntax) { + if (sql == null) { + return null; + } + String redacted = null; + for (Reading reading : readings(sql, syntax == null ? Syntax.UNKNOWN : syntax)) { + String candidate = reading.redact(sql); + if (candidate == null || (redacted != null && !redacted.equals(candidate))) { + return WITHHELD; + } + redacted = candidate; + } + return redacted; + } + + /** + * The readings a statement has to agree under. A reading that depends on a character the + * statement does not contain is the same as the one that does not, so it is left out. + */ + private static List<Reading> readings(String sql, Syntax syntax) { + boolean[] backslash = sql.indexOf('\\') >= 0 ? EITHER : NO; + int comment = sql.indexOf("/*"); + boolean[] nesting = comment >= 0 && sql.indexOf("/*", comment + 2) >= 0 ? EITHER : NO; + List<Reading> readings = new ArrayList<>(); + if (syntax == Syntax.STANDARD || syntax == Syntax.UNKNOWN) { + for (boolean backslashEscapes : backslash) { + for (boolean nestedComments : nesting) { + readings.add(new Reading(Syntax.STANDARD, backslashEscapes, nestedComments)); + } + } + } + if (syntax == Syntax.MYSQL || syntax == Syntax.UNKNOWN) { + for (boolean backslashEscapes : backslash) { + readings.add(new Reading(Syntax.MYSQL, backslashEscapes, false)); + } + } + if (syntax == Syntax.TRANSACT_SQL || syntax == Syntax.UNKNOWN) { + readings.add(new Reading(Syntax.TRANSACT_SQL, false, true)); + } + if (syntax == Syntax.GOOGLE_SQL || syntax == Syntax.UNKNOWN) { + readings.add(new Reading(Syntax.GOOGLE_SQL, true, false)); + } + return readings; + } + + /** + * One way of reading a statement. MySQL and GoogleSQL delimit identifiers with backticks and start a + * comment with {@code #}; only MySQL needs whitespace after {@code --}; only Transact-SQL delimits + * identifiers with square brackets. Outside standard SQL, double quotes can delimit a string: in + * MySQL unless {@code ANSI_QUOTES} is set, in Transact-SQL when {@code QUOTED_IDENTIFIER} is off, and + * in GoogleSQL always. Either way the quoted run ends at the same place, so it is replaced as a + * value. A GoogleSQL string is read on its own terms: a quote written twice ends it, a backslash + * escapes, and a quote written three times opens a string that ends only at the next three. + * + * @param syntax the syntax read in, never {@link Syntax#UNKNOWN} + * @param backslashEscapes whether a backslash escapes the character after it in a string + * @param nestedComments whether a block comment can contain another + */ + private record Reading(Syntax syntax, boolean backslashEscapes, boolean nestedComments) { + + /** + * @return the redacted statement, or null when a literal, a quoted identifier or a comment never ends + */ + String redact(String sql) { + boolean mysql = syntax == Syntax.MYSQL; + boolean googleSql = syntax == Syntax.GOOGLE_SQL; + boolean doubleQuotedValue = syntax != Syntax.STANDARD; + int length = sql.length(); + StringBuilder out = new StringBuilder(length); + int i = 0; + while (i < length) { + char c = sql.charAt(i); + char next = i + 1 < length ? sql.charAt(i + 1) : 0; + char previous = i > 0 ? sql.charAt(i - 1) : 0; + int dollarTag = c == '$' && !isIdentifierPart(previous) ? dollarTagLength(sql, i) : 0; + int end; + if (googleSql && (c == '\'' || c == '"')) { + end = endOfGoogleSqlString(sql, i); + out.append(PLACEHOLDER); + } + else if (c == '\'') { + end = endOfQuotedLiteral(sql, i); + out.append(PLACEHOLDER); + } + else if (c == '"') { + end = endOfQuoted(sql, i, '"', mysql && backslashEscapes, true); + if (end >= 0) { + out.append(doubleQuotedValue ? PLACEHOLDER : sql.substring(i, end)); + } + } + else if (c == '`' && (mysql || googleSql)) { + end = endOfQuoted(sql, i, '`', googleSql, mysql); + if (end >= 0) { + out.append(sql, i, end); + } + } + else if (c == '[' && syntax == Syntax.TRANSACT_SQL) { + end = endOfQuoted(sql, i, ']', false, true); + if (end >= 0) { + out.append(sql, i, end); + } + } + else if (c == '/' && next == '*') { + end = endOfBlockComment(sql, i); + out.append(' '); + } + else if ((c == '-' && next == '-' && (!mysql || i + 2 >= length || Character.isWhitespace(sql.charAt(i + 2)))) || + (c == '#' && (mysql || googleSql))) { + end = endOfLine(sql, i); + out.append(' '); + } + else if (dollarTag > 0) { + String tag = sql.substring(i, i + dollarTag); + int close = sql.indexOf(tag, i + dollarTag); + end = close < 0 ? -1 : close + tag.length(); + out.append(PLACEHOLDER); + } + else if ((isDigit(c) || (c == '.' && isDigit(next))) && !isIdentifierPart(previous)) { + end = endOfNumber(sql, i); + out.append(PLACEHOLDER); + } + else { + out.append(c); + end = i + 1; + } + if (end < 0) { + return null; + } + i = end; + } + return out.toString(); + } + + /** + * Ends a single-quoted literal, which may carry a prefix: {@code E'...'} always takes backslash + * escapes, and Oracle's {@code q'[...]'} ends at its own closing delimiter. + */ + private int endOfQuotedLiteral(String sql, int start) { + char prefix = start > 0 ? sql.charAt(start - 1) : 0; + boolean standalonePrefix = start < 2 || !isIdentifierPart(sql.charAt(start - 2)); + if ((prefix == 'q' || prefix == 'Q') && (standalonePrefix || isNationalPrefix(sql, start - 2)) && + start + 1 < sql.length() && !Character.isWhitespace(sql.charAt(start + 1)) && + sql.charAt(start + 1) != '\'') { + int close = sql.indexOf(closingDelimiter(sql.charAt(start + 1)) + "'", start + 2); + return close < 0 ? -1 : close + 2; + } + boolean escapeString = (prefix == 'e' || prefix == 'E') && standalonePrefix; + return endOfQuoted(sql, start, '\'', backslashEscapes || escapeString, true); + } + + private int endOfBlockComment(String sql, int start) { + int depth = 1; + int i = start + 2; + while (i + 1 < sql.length()) { + char c = sql.charAt(i); + char next = sql.charAt(i + 1); + if (c == '*' && next == '/') { + depth--; + i += 2; + if (depth == 0) { + return i; + } + } + else if (nestedComments && c == '/' && next == '*') { + depth++; + i += 2; + } + else { + i++; + } + } + return -1; + } + } + + /** + * Ends a quoted run. + * + * @param backslashEscapes whether a backslash escapes the character after it + * @param doubledClose whether the closing quote written twice stands for itself + * @return the index after the closing quote, or -1 when there is none + */ + private static int endOfQuoted(String sql, int start, char close, boolean backslashEscapes, boolean doubledClose) { + int i = start + 1; + while (i < sql.length()) { + char c = sql.charAt(i); + if (backslashEscapes && c == '\\') { + i += 2; + } + else if (c == close) { + if (doubledClose && i + 1 < sql.length() && sql.charAt(i + 1) == close) { + i += 2; + } + else { + return i + 1; + } + } + else { + i++; + } + } + return -1; + } + + /** + * Ends a GoogleSQL string, which a backslash escapes in and a quote written twice does not: that + * is an empty string, or the end of one string and the start of the next. A string that opens + * with its quote written three times is triple-quoted: it holds any quote, and ends at its quote + * written three times again. + * + * @return the index after the closing quote, or -1 when there is none + */ + private static int endOfGoogleSqlString(String sql, int start) { + char quote = sql.charAt(start); + if (start + 2 < sql.length() && sql.charAt(start + 1) == quote && sql.charAt(start + 2) == quote) { + return endOfTripleQuoted(sql, start, quote); + } + return endOfQuoted(sql, start, quote, true, false); + } + + private static int endOfTripleQuoted(String sql, int start, char quote) { + int i = start + 3; + while (i < sql.length()) { + char c = sql.charAt(i); + if (c == '\\') { + i += 2; + } + else if (c == quote && i + 2 < sql.length() && sql.charAt(i + 1) == quote && sql.charAt(i + 2) == quote) { + return i + 3; + } + else { + i++; + } + } + return -1; + } + + private static int endOfLine(String sql, int start) { + int i = start; + while (i < sql.length() && sql.charAt(i) != '\n' && sql.charAt(i) != '\r') { + i++; + } + return i; + } + + /** + * @return the length of the {@code $tag$} that opens a dollar-quoted string at the index, or 0 + * when there is none: a dollar followed by a digit is a positional parameter + */ + private static int dollarTagLength(String sql, int start) { + int i = start + 1; + while (i < sql.length()) { + char c = sql.charAt(i); + if (c == '$') { + return i + 1 - start; + } + boolean letter = Character.isLetter(c) || c == '_'; + if (!letter && (i == start + 1 || !isDigit(c))) { + return 0; + } + i++; + } + return 0; + } + + /** + * Ends a number: digits with a fraction and an exponent, or a hexadecimal number. Letters that + * follow it belong to it, so that nothing of a malformed number is left behind. + */ + private static int endOfNumber(String sql, int start) { + int length = sql.length(); + int i = start; + if (sql.charAt(i) == '0' && i + 1 < length && (sql.charAt(i + 1) == 'x' || sql.charAt(i + 1) == 'X')) { + i += 2; + } + else { + while (i < length && isDigit(sql.charAt(i))) { + i++; + } + if (i < length && sql.charAt(i) == '.') { + i++; + while (i < length && isDigit(sql.charAt(i))) { + i++; + } + } + if (i < length && (sql.charAt(i) == 'e' || sql.charAt(i) == 'E')) { + int exponent = i + 1 < length && (sql.charAt(i + 1) == '+' || sql.charAt(i + 1) == '-') ? i + 2 : i + 1; + if (exponent < length && isDigit(sql.charAt(exponent))) { + i = exponent; + } + } + } + while (i < length && isIdentifierPart(sql.charAt(i))) { + i++; + } + return i; + } + + private static boolean isNationalPrefix(String sql, int index) { + return index >= 0 && (sql.charAt(index) == 'n' || sql.charAt(index) == 'N') && + (index == 0 || !isIdentifierPart(sql.charAt(index - 1))); + } + + private static String closingDelimiter(char open) { + return switch (open) { + case '[' -> "]"; + case '(' -> ")"; + case '{' -> "}"; + case '<' -> ">"; + default -> String.valueOf(open); + }; + } + + private static boolean isDigit(char c) { + return c >= '0' && c <= '9'; + } + + private static boolean isIdentifierPart(char c) { Review Comment: Fixed in 0b6079cff3. Whether `#` and `@` are part of a name (Oracle, DB2, Transact-SQL) or operators (PostgreSQL) is now a reading of standard SQL, taken both ways when a statement contains one, so `select account_id#987654321 from account` and `select @987654321 from t` are withheld under `STANDARD`, while `t.col#` and `emp@remote`, which read the same either way, are still recorded. In `TRANSACT_SQL` a dollar before digits is an amount: `$987654321`, `$.5` and `-$12.34` record as `?`, `?`, `-?`; `a$1`, `@v` and `#temp` stay names, and `account_id#987654321` stays the name it is there. A dollar before digits stays a parameter in standard SQL and a name in MySQL, so `select $987654321` is withheld before the dialect is known. Rows for each in `SqlTailRedactorSpec`. ########## grails-datamapping-core/src/cli/groovy/org/grails/datastore/gorm/jdbc/sqltail/cli/SqlTailFileReader.groovy: ########## @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.jdbc.sqltail.cli + +import java.nio.charset.StandardCharsets + +import groovy.transform.CompileStatic + +import org.grails.datastore.gorm.jdbc.sqltail.SqlTailRecord + +/** + * Follows a capture file the way {@code tail -f} does: each poll returns the records written + * since the last one. A line that has no terminator yet is left for the next poll, so a record + * the application is still writing is never read in half. An application that starts again + * truncates the file and names its new run in the first line; when that line names another run, + * or the file shrinks, reading starts over from the beginning. The run tells a restart apart from + * a file that grew past where the reader was before it was next polled. A file truncated between + * being measured and being read, so that the read comes up short, is a restart too: it is read again + * from the beginning in the same poll. + * + * @since 8.1 + */ +@CompileStatic +class SqlTailFileReader implements Closeable { + + private static final byte NEWLINE = (byte) 10 + + /** How much of the file the first line is looked for in; a header is far shorter. */ + private static final int HEADER_LIMIT = 256 + + final File file + private long position + private String run + private boolean restarted + + /** + * @param file the capture file + * @param fromStart whether to read what the file already holds, or only what is written from now on + */ + SqlTailFileReader(File file, boolean fromStart) { + this.file = file + this.position = fromStart ? 0 : file.length() + this.run = file.exists() ? readRun() : null + } + + /** + * @return the records written since the last poll, or since the reader was opened + */ + List<SqlTailRecord> poll() { + restarted = false + // a read that comes up short is retried once, from the start; the next poll takes it from there + for (int attempt = 0; attempt < 2; attempt++) { + long length = file.length() + String current = length > 0 ? readRun() : null + if (length < position || (current != null && run != null && current != run)) { + position = 0 + restarted = true + } + if (current != null || restarted) { + // a file truncated before its header was written names its run on a later poll + run = current + } + if (length <= position) { + return [] + } + byte[] bytes = readFrom(position, length) + if (bytes.length == length - position) { + return recordsIn(bytes) + } + // the application truncated the file after it was measured: it has started again + position = 0 + restarted = true + run = null + } + [] + } + + /** + * @return whether the last poll found a new run in the file, which an application restart writes + */ + boolean isRestarted() { + restarted + } + + /** + * @return how far the file has been read + */ + long getPosition() { + position + } + + /** + * @return the run that wrote the file, as its first line names it; null before the line is complete + */ + String getRun() { + run + } + + @Override + void close() { + // the file is opened and closed per poll + } + + /** + * Takes the complete lines out of what was read, and leaves a line still being written for the next poll. + */ + private List<SqlTailRecord> recordsIn(byte[] bytes) { + int consumed = lastLineEnd(bytes) + if (consumed == 0) { + return [] + } + position += consumed + String text = new String(bytes, 0, consumed, StandardCharsets.UTF_8) + List<SqlTailRecord> records = [] + for (String line : text.split('\n')) { + SqlTailRecord record = SqlTailRecord.parse(line) + if (record != null) { + records.add(record) + } + } + records + } + + private static int lastLineEnd(byte[] bytes) { + for (int i = bytes.length - 1; i >= 0; i--) { + if (bytes[i] == NEWLINE) { + return i + 1 + } + } + 0 + } + + private String readRun() { + byte[] head = readFrom(0, Math.min(file.length(), (long) HEADER_LIMIT)) + for (int i = 0; i < head.length; i++) { + if (head[i] == NEWLINE) { + return SqlTailRecord.parseHeader(new String(head, 0, i, StandardCharsets.UTF_8)) + } + } + null + } + + /** + * @return the bytes between the offsets, or fewer when the file ends before the second one or is + * gone: it was truncated, or removed, after it was measured + */ + private byte[] readFrom(long from, long to) { Review Comment: Fixed in 0b6079cff3. A poll now reads at most a chunk (1 MiB, `SqlTailFileReader.CHUNK`) and reports `behind` while the file holds more; the runner polls on without pausing while it does, and `--once` prints its summary only once the reader has caught up. A chunk without a line end is a line longer than a chunk: it is dropped up to its end, as a line that is not a record is, so the reader never allocates more than a chunk. `SqlTailFileReaderSpec` covers reading in chunks and the over-long line with a small chunk, and `SqlTailRunnerSpec` has `--once` on a capture several chunks long reporting every record. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
