This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new c14ade2b91 Upgrade protobuf definitions to flightsql 17.0 (#6133)
(#6169)
c14ade2b91 is described below
commit c14ade2b917889a55d4b098b4a48b36ddb2c48c7
Author: Andrew Lamb <[email protected]>
AuthorDate: Thu Aug 1 07:05:10 2024 -0400
Upgrade protobuf definitions to flightsql 17.0 (#6133) (#6169)
* Update FlightSql.proto to version 17.0
Adds new message CommandStatementIngest and removes `experimental` from
other messages.
* Regenerate flight sql protocol
This upgrades the file to version 17.0 of the protobuf definition.
Co-authored-by: Douglas Anderson <[email protected]>
---
arrow-flight/src/sql/arrow.flight.protocol.sql.rs | 177 +++++++++++++++++++++-
format/FlightSql.proto | 112 +++++++++-----
2 files changed, 252 insertions(+), 37 deletions(-)
diff --git a/arrow-flight/src/sql/arrow.flight.protocol.sql.rs
b/arrow-flight/src/sql/arrow.flight.protocol.sql.rs
index 5e6f198df7..3eeed6ff4b 100644
--- a/arrow-flight/src/sql/arrow.flight.protocol.sql.rs
+++ b/arrow-flight/src/sql/arrow.flight.protocol.sql.rs
@@ -798,9 +798,157 @@ pub struct CommandPreparedStatementUpdate {
pub prepared_statement_handle: ::prost::bytes::Bytes,
}
///
-/// Returned from the RPC call DoPut when a CommandStatementUpdate
-/// CommandPreparedStatementUpdate was in the request, containing
-/// results from the update.
+/// Represents a bulk ingestion request. Used in the command member of
FlightDescriptor
+/// for the the RPC call DoPut to cause the server load the contents of the
stream's
+/// FlightData into the target destination.
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct CommandStatementIngest {
+ /// The behavior for handling the table definition.
+ #[prost(message, optional, tag = "1")]
+ pub table_definition_options: ::core::option::Option<
+ command_statement_ingest::TableDefinitionOptions,
+ >,
+ /// The table to load data into.
+ #[prost(string, tag = "2")]
+ pub table: ::prost::alloc::string::String,
+ /// The db_schema of the destination table to load data into. If unset, a
backend-specific default may be used.
+ #[prost(string, optional, tag = "3")]
+ pub schema: ::core::option::Option<::prost::alloc::string::String>,
+ /// The catalog of the destination table to load data into. If unset, a
backend-specific default may be used.
+ #[prost(string, optional, tag = "4")]
+ pub catalog: ::core::option::Option<::prost::alloc::string::String>,
+ ///
+ /// Store ingested data in a temporary table.
+ /// The effect of setting temporary is to place the table in a
backend-defined namespace, and to drop the table at the end of the session.
+ /// The namespacing may make use of a backend-specific schema and/or
catalog.
+ /// The server should return an error if an explicit choice of schema or
catalog is incompatible with the server's namespacing decision.
+ #[prost(bool, tag = "5")]
+ pub temporary: bool,
+ /// Perform the ingestion as part of this transaction. If specified,
results should not be committed in the event of an error/cancellation.
+ #[prost(bytes = "bytes", optional, tag = "6")]
+ pub transaction_id: ::core::option::Option<::prost::bytes::Bytes>,
+ /// Backend-specific options.
+ #[prost(map = "string, string", tag = "1000")]
+ pub options: ::std::collections::HashMap<
+ ::prost::alloc::string::String,
+ ::prost::alloc::string::String,
+ >,
+}
+/// Nested message and enum types in `CommandStatementIngest`.
+pub mod command_statement_ingest {
+ /// Options for table definition behavior
+ #[allow(clippy::derive_partial_eq_without_eq)]
+ #[derive(Clone, Copy, PartialEq, ::prost::Message)]
+ pub struct TableDefinitionOptions {
+ #[prost(
+ enumeration = "table_definition_options::TableNotExistOption",
+ tag = "1"
+ )]
+ pub if_not_exist: i32,
+ #[prost(enumeration = "table_definition_options::TableExistsOption",
tag = "2")]
+ pub if_exists: i32,
+ }
+ /// Nested message and enum types in `TableDefinitionOptions`.
+ pub mod table_definition_options {
+ /// The action to take if the target table does not exist
+ #[derive(
+ Clone,
+ Copy,
+ Debug,
+ PartialEq,
+ Eq,
+ Hash,
+ PartialOrd,
+ Ord,
+ ::prost::Enumeration
+ )]
+ #[repr(i32)]
+ pub enum TableNotExistOption {
+ /// Do not use. Servers should error if this is specified by a
client.
+ Unspecified = 0,
+ /// Create the table if it does not exist
+ Create = 1,
+ /// Fail if the table does not exist
+ Fail = 2,
+ }
+ impl TableNotExistOption {
+ /// String value of the enum field names used in the ProtoBuf
definition.
+ ///
+ /// The values are not transformed in any way and thus are
considered stable
+ /// (if the ProtoBuf definition does not change) and safe for
programmatic use.
+ pub fn as_str_name(&self) -> &'static str {
+ match self {
+ TableNotExistOption::Unspecified => {
+ "TABLE_NOT_EXIST_OPTION_UNSPECIFIED"
+ }
+ TableNotExistOption::Create =>
"TABLE_NOT_EXIST_OPTION_CREATE",
+ TableNotExistOption::Fail => "TABLE_NOT_EXIST_OPTION_FAIL",
+ }
+ }
+ /// Creates an enum from field names used in the ProtoBuf
definition.
+ pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+ match value {
+ "TABLE_NOT_EXIST_OPTION_UNSPECIFIED" =>
Some(Self::Unspecified),
+ "TABLE_NOT_EXIST_OPTION_CREATE" => Some(Self::Create),
+ "TABLE_NOT_EXIST_OPTION_FAIL" => Some(Self::Fail),
+ _ => None,
+ }
+ }
+ }
+ /// The action to take if the target table already exists
+ #[derive(
+ Clone,
+ Copy,
+ Debug,
+ PartialEq,
+ Eq,
+ Hash,
+ PartialOrd,
+ Ord,
+ ::prost::Enumeration
+ )]
+ #[repr(i32)]
+ pub enum TableExistsOption {
+ /// Do not use. Servers should error if this is specified by a
client.
+ Unspecified = 0,
+ /// Fail if the table already exists
+ Fail = 1,
+ /// Append to the table if it already exists
+ Append = 2,
+ /// Drop and recreate the table if it already exists
+ Replace = 3,
+ }
+ impl TableExistsOption {
+ /// String value of the enum field names used in the ProtoBuf
definition.
+ ///
+ /// The values are not transformed in any way and thus are
considered stable
+ /// (if the ProtoBuf definition does not change) and safe for
programmatic use.
+ pub fn as_str_name(&self) -> &'static str {
+ match self {
+ TableExistsOption::Unspecified =>
"TABLE_EXISTS_OPTION_UNSPECIFIED",
+ TableExistsOption::Fail => "TABLE_EXISTS_OPTION_FAIL",
+ TableExistsOption::Append => "TABLE_EXISTS_OPTION_APPEND",
+ TableExistsOption::Replace =>
"TABLE_EXISTS_OPTION_REPLACE",
+ }
+ }
+ /// Creates an enum from field names used in the ProtoBuf
definition.
+ pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+ match value {
+ "TABLE_EXISTS_OPTION_UNSPECIFIED" =>
Some(Self::Unspecified),
+ "TABLE_EXISTS_OPTION_FAIL" => Some(Self::Fail),
+ "TABLE_EXISTS_OPTION_APPEND" => Some(Self::Append),
+ "TABLE_EXISTS_OPTION_REPLACE" => Some(Self::Replace),
+ _ => None,
+ }
+ }
+ }
+ }
+}
+///
+/// Returned from the RPC call DoPut when a CommandStatementUpdate,
+/// CommandPreparedStatementUpdate, or CommandStatementIngest was
+/// in the request, containing results from the update.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct DoPutUpdateResult {
@@ -972,6 +1120,19 @@ pub enum SqlInfo {
/// query cancellation (the CancelQuery action).
FlightSqlServerCancel = 9,
///
+ /// Retrieves a boolean value indicating whether the Flight SQL Server
supports executing
+ /// bulk ingestion.
+ FlightSqlServerBulkIngestion = 10,
+ ///
+ /// Retrieves a boolean value indicating whether transactions are
supported for bulk ingestion. If not, invoking
+ /// the method commit in the context of a bulk ingestion is a noop, and
the isolation level is
+ ///
`arrow.flight.protocol.sql.SqlTransactionIsolationLevel.TRANSACTION_NONE`.
+ ///
+ /// Returns:
+ /// - false: if bulk ingestion transactions are unsupported;
+ /// - true: if bulk ingestion transactions are supported.
+ FlightSqlServerIngestTransactionsSupported = 11,
+ ///
/// Retrieves an int32 indicating the timeout (in milliseconds) for
prepared statement handles.
///
/// If 0, there is no timeout. Servers should reset the timeout when the
handle is used in a command.
@@ -1542,6 +1703,10 @@ impl SqlInfo {
}
SqlInfo::FlightSqlServerTransaction =>
"FLIGHT_SQL_SERVER_TRANSACTION",
SqlInfo::FlightSqlServerCancel => "FLIGHT_SQL_SERVER_CANCEL",
+ SqlInfo::FlightSqlServerBulkIngestion =>
"FLIGHT_SQL_SERVER_BULK_INGESTION",
+ SqlInfo::FlightSqlServerIngestTransactionsSupported => {
+ "FLIGHT_SQL_SERVER_INGEST_TRANSACTIONS_SUPPORTED"
+ }
SqlInfo::FlightSqlServerStatementTimeout => {
"FLIGHT_SQL_SERVER_STATEMENT_TIMEOUT"
}
@@ -1674,6 +1839,12 @@ impl SqlInfo {
}
"FLIGHT_SQL_SERVER_TRANSACTION" =>
Some(Self::FlightSqlServerTransaction),
"FLIGHT_SQL_SERVER_CANCEL" => Some(Self::FlightSqlServerCancel),
+ "FLIGHT_SQL_SERVER_BULK_INGESTION" => {
+ Some(Self::FlightSqlServerBulkIngestion)
+ }
+ "FLIGHT_SQL_SERVER_INGEST_TRANSACTIONS_SUPPORTED" => {
+ Some(Self::FlightSqlServerIngestTransactionsSupported)
+ }
"FLIGHT_SQL_SERVER_STATEMENT_TIMEOUT" => {
Some(Self::FlightSqlServerStatementTimeout)
}
diff --git a/format/FlightSql.proto b/format/FlightSql.proto
index 4fc68f2a5d..8f9e1c8d82 100644
--- a/format/FlightSql.proto
+++ b/format/FlightSql.proto
@@ -43,7 +43,6 @@
* where there is one row per requested piece of metadata information.
*/
message CommandGetSqlInfo {
- option (experimental) = true;
/*
* Values are modelled after ODBC's SQLGetInfo() function. This information
is intended to provide
@@ -135,6 +134,23 @@
*/
FLIGHT_SQL_SERVER_CANCEL = 9;
+ /*
+ * Retrieves a boolean value indicating whether the Flight SQL Server
supports executing
+ * bulk ingestion.
+ */
+ FLIGHT_SQL_SERVER_BULK_INGESTION = 10;
+
+ /*
+ * Retrieves a boolean value indicating whether transactions are supported
for bulk ingestion. If not, invoking
+ * the method commit in the context of a bulk ingestion is a noop, and the
isolation level is
+ * `arrow.flight.protocol.sql.SqlTransactionIsolationLevel.TRANSACTION_NONE`.
+ *
+ * Returns:
+ * - false: if bulk ingestion transactions are unsupported;
+ * - true: if bulk ingestion transactions are supported.
+ */
+ FLIGHT_SQL_SERVER_INGEST_TRANSACTIONS_SUPPORTED = 11;
+
/*
* Retrieves an int32 indicating the timeout (in milliseconds) for prepared
statement handles.
*
@@ -1114,7 +1130,6 @@
* The returned data should be ordered by data_type and then by type_name.
*/
message CommandGetXdbcTypeInfo {
- option (experimental) = true;
/*
* Specifies the data type to search for the info.
@@ -1136,7 +1151,6 @@
* The returned data should be ordered by catalog_name.
*/
message CommandGetCatalogs {
- option (experimental) = true;
}
/*
@@ -1154,7 +1168,6 @@
* The returned data should be ordered by catalog_name, then db_schema_name.
*/
message CommandGetDbSchemas {
- option (experimental) = true;
/*
* Specifies the Catalog to search for the tables.
@@ -1202,7 +1215,6 @@
* The returned data should be ordered by catalog_name, db_schema_name,
table_name, then table_type, followed by table_schema if requested.
*/
message CommandGetTables {
- option (experimental) = true;
/*
* Specifies the Catalog to search for the tables.
@@ -1255,7 +1267,6 @@
* The returned data should be ordered by table_type.
*/
message CommandGetTableTypes {
- option (experimental) = true;
}
/*
@@ -1276,7 +1287,6 @@
* The returned data should be ordered by catalog_name, db_schema_name,
table_name, key_name, then key_sequence.
*/
message CommandGetPrimaryKeys {
- option (experimental) = true;
/*
* Specifies the catalog to search for the table.
@@ -1331,7 +1341,6 @@
* update_rule and delete_rule returns a byte that is equivalent to actions
declared on UpdateDeleteRules enum.
*/
message CommandGetExportedKeys {
- option (experimental) = true;
/*
* Specifies the catalog to search for the foreign key table.
@@ -1382,7 +1391,6 @@
* - 4 = SET DEFAULT
*/
message CommandGetImportedKeys {
- option (experimental) = true;
/*
* Specifies the catalog to search for the primary key table.
@@ -1435,7 +1443,6 @@
* - 4 = SET DEFAULT
*/
message CommandGetCrossReference {
- option (experimental) = true;
/**
* The catalog name where the parent table is.
@@ -1482,7 +1489,6 @@
* Request message for the "CreatePreparedStatement" action on a Flight SQL
enabled backend.
*/
message ActionCreatePreparedStatementRequest {
- option (experimental) = true;
// The valid SQL string to create a prepared statement for.
string query = 1;
@@ -1495,7 +1501,6 @@
* An embedded message describing a Substrait plan to execute.
*/
message SubstraitPlan {
- option (experimental) = true;
// The serialized substrait.Plan to create a prepared statement for.
// XXX(ARROW-16902): this is bytes instead of an embedded message
@@ -1512,7 +1517,6 @@
* Request message for the "CreatePreparedSubstraitPlan" action on a Flight
SQL enabled backend.
*/
message ActionCreatePreparedSubstraitPlanRequest {
- option (experimental) = true;
// The serialized substrait.Plan to create a prepared statement for.
SubstraitPlan plan = 1;
@@ -1531,7 +1535,6 @@
* The result should be wrapped in a google.protobuf.Any message.
*/
message ActionCreatePreparedStatementResult {
- option (experimental) = true;
// Opaque handle for the prepared statement on the server.
bytes prepared_statement_handle = 1;
@@ -1553,7 +1556,6 @@
* Closes server resources associated with the prepared statement handle.
*/
message ActionClosePreparedStatementRequest {
- option (experimental) = true;
// Opaque handle for the prepared statement on the server.
bytes prepared_statement_handle = 1;
@@ -1564,7 +1566,6 @@
* Begins a transaction.
*/
message ActionBeginTransactionRequest {
- option (experimental) = true;
}
/*
@@ -1575,7 +1576,6 @@
* FLIGHT_SQL_TRANSACTION_SUPPORT_SAVEPOINT.
*/
message ActionBeginSavepointRequest {
- option (experimental) = true;
// The transaction to which a savepoint belongs.
bytes transaction_id = 1;
@@ -1593,7 +1593,6 @@
* The result should be wrapped in a google.protobuf.Any message.
*/
message ActionBeginTransactionResult {
- option (experimental) = true;
// Opaque handle for the transaction on the server.
bytes transaction_id = 1;
@@ -1609,7 +1608,6 @@
* The result should be wrapped in a google.protobuf.Any message.
*/
message ActionBeginSavepointResult {
- option (experimental) = true;
// Opaque handle for the savepoint on the server.
bytes savepoint_id = 1;
@@ -1624,7 +1622,6 @@
* invalidated, as are all associated savepoints.
*/
message ActionEndTransactionRequest {
- option (experimental) = true;
enum EndTransaction {
END_TRANSACTION_UNSPECIFIED = 0;
@@ -1650,7 +1647,6 @@
* savepoints created after the current savepoint.
*/
message ActionEndSavepointRequest {
- option (experimental) = true;
enum EndSavepoint {
END_SAVEPOINT_UNSPECIFIED = 0;
@@ -1685,7 +1681,6 @@
* - GetFlightInfo: execute the query.
*/
message CommandStatementQuery {
- option (experimental) = true;
// The SQL syntax.
string query = 1;
@@ -1712,7 +1707,6 @@
* - DoPut: execute the query.
*/
message CommandStatementSubstraitPlan {
- option (experimental) = true;
// A serialized substrait.Plan
SubstraitPlan plan = 1;
@@ -1725,7 +1719,6 @@
* This should be used only once and treated as an opaque value, that is,
clients should not attempt to parse this.
*/
message TicketStatementQuery {
- option (experimental) = true;
// Unique identifier for the instance of the statement to execute.
bytes statement_handle = 1;
@@ -1753,7 +1746,6 @@
* - GetFlightInfo: execute the prepared statement instance.
*/
message CommandPreparedStatementQuery {
- option (experimental) = true;
// Opaque handle for the prepared statement on the server.
bytes prepared_statement_handle = 1;
@@ -1764,7 +1756,6 @@
* for the RPC call DoPut to cause the server to execute the included SQL
update.
*/
message CommandStatementUpdate {
- option (experimental) = true;
// The SQL syntax.
string query = 1;
@@ -1778,19 +1769,75 @@
* prepared statement handle as an update.
*/
message CommandPreparedStatementUpdate {
- option (experimental) = true;
// Opaque handle for the prepared statement on the server.
bytes prepared_statement_handle = 1;
}
/*
- * Returned from the RPC call DoPut when a CommandStatementUpdate
- * CommandPreparedStatementUpdate was in the request, containing
- * results from the update.
+ * Represents a bulk ingestion request. Used in the command member of
FlightDescriptor
+ * for the the RPC call DoPut to cause the server load the contents of the
stream's
+ * FlightData into the target destination.
+ */
+message CommandStatementIngest {
+
+ // Options for table definition behavior
+ message TableDefinitionOptions {
+ // The action to take if the target table does not exist
+ enum TableNotExistOption {
+ // Do not use. Servers should error if this is specified by a client.
+ TABLE_NOT_EXIST_OPTION_UNSPECIFIED = 0;
+ // Create the table if it does not exist
+ TABLE_NOT_EXIST_OPTION_CREATE = 1;
+ // Fail if the table does not exist
+ TABLE_NOT_EXIST_OPTION_FAIL = 2;
+ }
+ // The action to take if the target table already exists
+ enum TableExistsOption {
+ // Do not use. Servers should error if this is specified by a client.
+ TABLE_EXISTS_OPTION_UNSPECIFIED = 0;
+ // Fail if the table already exists
+ TABLE_EXISTS_OPTION_FAIL = 1;
+ // Append to the table if it already exists
+ TABLE_EXISTS_OPTION_APPEND = 2;
+ // Drop and recreate the table if it already exists
+ TABLE_EXISTS_OPTION_REPLACE = 3;
+ }
+
+ TableNotExistOption if_not_exist = 1;
+ TableExistsOption if_exists = 2;
+ }
+
+ // The behavior for handling the table definition.
+ TableDefinitionOptions table_definition_options = 1;
+ // The table to load data into.
+ string table = 2;
+ // The db_schema of the destination table to load data into. If unset, a
backend-specific default may be used.
+ optional string schema = 3;
+ // The catalog of the destination table to load data into. If unset, a
backend-specific default may be used.
+ optional string catalog = 4;
+ /*
+ * Store ingested data in a temporary table.
+ * The effect of setting temporary is to place the table in a
backend-defined namespace, and to drop the table at the end of the session.
+ * The namespacing may make use of a backend-specific schema and/or catalog.
+ * The server should return an error if an explicit choice of schema or
catalog is incompatible with the server's namespacing decision.
+ */
+ bool temporary = 5;
+ // Perform the ingestion as part of this transaction. If specified, results
should not be committed in the event of an error/cancellation.
+ optional bytes transaction_id = 6;
+
+ // Future extensions to the parameters of CommandStatementIngest should be
added here, at a lower index than the generic 'options' parameter.
+
+ // Backend-specific options.
+ map<string, string> options = 1000;
+}
+
+/*
+ * Returned from the RPC call DoPut when a CommandStatementUpdate,
+ * CommandPreparedStatementUpdate, or CommandStatementIngest was
+ * in the request, containing results from the update.
*/
message DoPutUpdateResult {
- option (experimental) = true;
// The number of records updated. A return value of -1 represents
// an unknown updated record count.
@@ -1804,7 +1851,6 @@
* can continue as though the fields in this message were not provided or set
to sensible default values.
*/
message DoPutPreparedStatementResult {
- option (experimental) = true;
// Represents a (potentially updated) opaque handle for the prepared
statement on the server.
// Because the handle could potentially be updated, any previous handles
for this prepared
@@ -1836,7 +1882,6 @@
*/
message ActionCancelQueryRequest {
option deprecated = true;
- option (experimental) = true;
// The result of the GetFlightInfo RPC that initiated the query.
// XXX(ARROW-16902): this must be a serialized FlightInfo, but is
@@ -1855,7 +1900,6 @@
*/
message ActionCancelQueryResult {
option deprecated = true;
- option (experimental) = true;
enum CancelResult {
// The cancellation status is unknown. Servers should avoid using
