FrankChen021 commented on code in PR #19698:
URL: https://github.com/apache/druid/pull/19698#discussion_r3698986553
##########
services/src/main/java/org/apache/druid/cli/ExportMetadata.java:
##########
@@ -386,20 +392,27 @@ private void rewriteSegmentsExport(
while ((line = reader.readLine()) != null) {
String[] parsed = PARSER.parseLine(line);
StringBuilder newLineBuilder = new StringBuilder();
- newLineBuilder.append(parsed[0]).append(","); //id
- newLineBuilder.append(parsed[1]).append(","); //dataSource
- newLineBuilder.append(parsed[2]).append(","); //created_date
- newLineBuilder.append(parsed[3]).append(","); //start
- newLineBuilder.append(parsed[4]).append(","); //end
+ newLineBuilder.append(csvEscapeField(parsed[0])).append(","); //id
+ newLineBuilder.append(csvEscapeField(parsed[1])).append(",");
//dataSource
+ newLineBuilder.append(csvEscapeField(parsed[2])).append(",");
//created_date
+ newLineBuilder.append(csvEscapeField(parsed[3])).append(","); //start
+ newLineBuilder.append(csvEscapeField(parsed[4])).append(","); //end
newLineBuilder.append(convertBooleanString(parsed[5])).append(",");
//partitioned
- newLineBuilder.append(parsed[6]).append(","); //version
+ newLineBuilder.append(csvEscapeField(parsed[6])).append(","); //version
newLineBuilder.append(convertBooleanString(parsed[7])).append(",");
//used
if (s3Bucket != null || hadoopStorageDirectory != null || newLocalPath
!= null) {
newLineBuilder.append(makePayloadWithConvertedLoadSpec(parsed[8]));
} else {
newLineBuilder.append(rewriteHexPayloadAsEscapedJson(parsed[8]));
//payload
}
+
+ // Preserve any additional columns after payload (e.g.
used_status_last_updated,
+ // indexing_state_fingerprint, upgraded_from_segment_id,
schema_fingerprint, num_rows)
+ for (int i = 9; i < parsed.length; i++) {
Review Comment:
[P1] Update imports for preserved segment columns
Current segment tables emit at least 12 fields, and this loop now retains
all of them, but the documented PostgreSQL and MySQL import commands still
declare only the original nine columns. PostgreSQL `COPY` rejects every such
row as extra data, while MySQL truncates or ignores trailing fields with
warnings. Emit a deterministic schema-compatible order and update import
commands to include all exported columns.
##########
server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java:
##########
@@ -1038,6 +1045,84 @@ public void createAuditTable()
}
}
+ @Override
+ public void exportTable(
+ final String tableName,
+ final String outputPath
+ )
+ {
+ exportTableWithJdbc(tableName, outputPath);
+ }
+
+ /**
+ * Exports a table to a CSV file using generic JDBC.
+ * Binary columns are hex-encoded and booleans are written as true/false
strings.
+ * Subclasses may override {@link #exportTable} with a database-specific
implementation
+ * while this method remains available for testing or fallback.
+ */
+ protected void exportTableWithJdbc(
+ final String tableName,
+ final String outputPath
+ )
+ {
+ // Use a transaction so that the connection has autoCommit=false.
+ // PostgreSQL JDBC requires autoCommit=false and a positive fetch size
+ // to use cursor-based streaming instead of buffering the entire ResultSet.
+ retryTransaction(
+ (TransactionCallback<Void>) (handle, status) -> {
+ final Connection conn = handle.getConnection();
+ try (Statement stmt = conn.createStatement()) {
+ final int fetchSize = getStreamingFetchSize();
+ if (fetchSize > 0) {
+ stmt.setFetchSize(fetchSize);
+ }
+ try (ResultSet rs =
stmt.executeQuery(StringUtils.format("SELECT * FROM %s", tableName));
+ FileOutputStream fos = new FileOutputStream(outputPath);
+ OutputStreamWriter writer = new OutputStreamWriter(fos,
StandardCharsets.UTF_8)) {
+ final ResultSetMetaData meta = rs.getMetaData();
+ final int columnCount = meta.getColumnCount();
+ while (rs.next()) {
+ for (int i = 1; i <= columnCount; i++) {
+ if (i > 1) {
+ writer.write(',');
+ }
+ final int colType = meta.getColumnType(i);
+ if (colType == Types.BINARY || colType == Types.VARBINARY
+ || colType == Types.LONGVARBINARY || colType ==
Types.BLOB
+ || (colType == Types.OTHER &&
"bytea".equalsIgnoreCase(meta.getColumnTypeName(i)))) {
+ final byte[] bytes = rs.getBytes(i);
+ if (bytes != null) {
+ writer.write(BaseEncoding.base16().encode(bytes));
+ }
+ } else if (colType == Types.BOOLEAN || colType ==
Types.BIT) {
+ final boolean val = rs.getBoolean(i);
+ if (!rs.wasNull()) {
+ writer.write(String.valueOf(val));
+ }
+ } else {
+ final String val = rs.getString(i);
+ if (val != null) {
+ if (val.contains(",") || val.contains("\"") ||
val.contains("\n") || val.contains("\r")) {
Review Comment:
[P2] Preserve backslashes through CSV rewrites
The writer emits RFC 4180 data and leaves backslashes unchanged, but
`ExportMetadata` reads it with default OpenCSV `CSVParser`, where backslash is
the escape character. Consequently, a valid identifier such as `foo\bar` is
parsed as `foobar` before `csvEscapeField` runs; Druid's ID validation
explicitly permits backslashes. Use an RFC 4180 reader/parser or configure a
null escape character, and add a round-trip test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
