This is an automated email from the ASF dual-hosted git repository.
curth pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new 298977a9b fix(csharp/src/Drivers/BigQuery): Adjust default dataset id
(#3187)
298977a9b is described below
commit 298977a9b7162bada9281b9b67ad65c854fb00c6
Author: davidhcoe <[email protected]>
AuthorDate: Wed Jul 23 20:52:52 2025 +0200
fix(csharp/src/Drivers/BigQuery): Adjust default dataset id (#3187)
- change to use _bqadbc_temp_tables as the default dataset if large
record sets are needed
- removes the created_by tag, which was having its own issues with the
`-SNAPSHOT+` content was included in the file version. Since the tables
are now in the _bqadbc_temp_tables dataset, it shouldn't be necessary to
indicate what driver created them
- adds some testing for parallelizing queries to do scale testing
Co-authored-by: David Coe <>
---
csharp/src/Drivers/BigQuery/BigQueryParameters.cs | 5 ++--
csharp/src/Drivers/BigQuery/BigQueryStatement.cs | 8 ++----
csharp/src/Drivers/BigQuery/readme.md | 6 ++---
.../Drivers/BigQuery/BigQueryTestConfiguration.cs | 25 ++++++++++++++++++
csharp/test/Drivers/BigQuery/DriverTests.cs | 30 ++++++++++++++++++++++
5 files changed, 62 insertions(+), 12 deletions(-)
diff --git a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs
b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs
index f0002c609..a02173e0d 100644
--- a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs
+++ b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs
@@ -91,9 +91,8 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
// default value per
https://pkg.go.dev/cloud.google.com/go/bigquery#section-readme
public const string DetectProjectId = "*detect-project-id*";
- // Reuse what the ODBC driver already has in place, in case a caller
- // has permission issues trying to create a new dataset
- public const string DefaultLargeDatasetId = "_bqodbc_temp_tables";
+ // matches the pattern for odbc, but for adbc
+ public const string DefaultLargeDatasetId = "_bqadbc_temp_tables";
public const string PublicProjectId = "bigquery-public-data";
}
diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs
b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs
index a148e0f22..e22a4815c 100644
--- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs
+++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs
@@ -485,15 +485,11 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
{
activity?.AddBigQueryTag("large_results.dataset.try_create", datasetId);
DatasetReference reference =
this.Client.GetDatasetReference(datasetId);
+
BigQueryDataset bigQueryDataset = new
BigQueryDataset(this.Client, new Dataset()
{
DatasetReference = reference,
- DefaultTableExpirationMs =
(long)TimeSpan.FromDays(1).TotalMilliseconds,
- Labels = new Dictionary<string, string>()
- {
- // lower case, no spaces or periods per
https://cloud.google.com/bigquery/docs/labels-intro
- { "created_by",
this.bigQueryConnection.DriverName.ToLowerInvariant().Replace(" ","_") + "_v_"
+ AssemblyVersion.Replace(".","_") }
- }
+ DefaultTableExpirationMs =
(long)TimeSpan.FromDays(1).TotalMilliseconds
});
dataset = this.Client.CreateDataset(datasetId,
bigQueryDataset.Resource);
diff --git a/csharp/src/Drivers/BigQuery/readme.md
b/csharp/src/Drivers/BigQuery/readme.md
index 13d2ec6cc..96605a9c2 100644
--- a/csharp/src/Drivers/BigQuery/readme.md
+++ b/csharp/src/Drivers/BigQuery/readme.md
@@ -93,7 +93,7 @@ The following parameters can be used to configure the driver
behavior. The param
Include the `bigquery-public-data` project ID with the
list of project IDs.
**adbc.bigquery.large_results_dataset**<br>
- Optional. Sets the dataset ID to use for large
results. The dataset needs to be in the same region as the data being queried.
If no value is specified, the driver will attempt to use or create
`_bqodbc_temp_tables`. A randomly generated table name will be used for the
DestinationTable.
+ Optional. Sets the dataset ID to use for large
results. The dataset needs to be in the same region as the data being queried.
If no value is specified, the driver will attempt to use or create
`_bqadbc_temp_tables`. A randomly generated table name will be used for the
DestinationTable.
**adbc.bigquery.large_results_destination_table**<br>
Optional. Sets the
[DestinationTable](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.QueryOptions#Google_Cloud_BigQuery_V2_QueryOptions_DestinationTable)
value of the QueryOptions if configured. Expects the format to be
`{projectId}.{datasetId}.{tableId}` to set the corresponding values in the
[TableReference](https://github.com/googleapis/google-api-dotnet-client/blob/6c415c73788b848711e47c6dd33c2f93c76faf9
[...]
@@ -169,8 +169,8 @@ a dataset using the `adbc.bigquery.large_results_dataset`
parameter.
Behavior:
- If a destination table is explicitly set, the driver will use that value.
- If only a dataset value is set, the driver will attempt to retrieve the
dataset. If the dataset does not exist, the driver will attempt to
- create it. The default table expiration will be set to 1 day and a
`created_by` label will be included with the driver name and version that
created the dataset. For example `created_by :
adbc_bigquery_driver_v_0_19_0_0`. A randomly generated name will be used for
the table name.
-- If a destination table and a dataset are not specified, the driver will
attempt to use or create the `_bqodbc_temp_tables` dataset using the same
defaults and label specified above. A randomly generated name will be used for
the table name.
+ create it. The default table expiration will be set to 1 day. A randomly
generated name will be used for the table name.
+- If a destination table and a dataset are not specified, the driver will
attempt to use or create the `_bqadbc_temp_tables` dataset using the same
defaults and label specified above. A randomly generated name will be used for
the table name.
## Permissions
diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs
b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs
index 17071465b..b88a8b7f2 100644
--- a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs
+++ b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs
@@ -36,6 +36,7 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
{
AllowLargeResults = false;
IncludeTableConstraints = true;
+ ParallelQueries = new List<ParallelQuery>();
}
[JsonPropertyName("projectId")]
@@ -130,6 +131,30 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
[JsonPropertyName("entraConfiguration")]
public EntraConfiguration? EntraConfiguration { get; set; }
+
+ /// <summary>
+ /// The number of times to repeat the parallel runs.
+ /// </summary>
+ [JsonPropertyName("numberOfParallelRuns")]
+ public int NumberOfParallelRuns { get; set; }
+
+ [JsonPropertyName("queries")]
+ public List<ParallelQuery> ParallelQueries { get; set; }
+ }
+
+ class ParallelQuery
+ {
+ /// <summary>
+ /// The query to run.
+ /// </summary>
+ [JsonPropertyName("query")]
+ public string Query { get; set; } = string.Empty;
+
+ /// <summary>
+ /// The number of expected results from the query.
+ /// </summary>
+ [JsonPropertyName("expectedResults")]
+ public long ExpectedResultsCount { get; set; }
}
class EntraConfiguration
diff --git a/csharp/test/Drivers/BigQuery/DriverTests.cs
b/csharp/test/Drivers/BigQuery/DriverTests.cs
index 15dafc0ec..c15458147 100644
--- a/csharp/test/Drivers/BigQuery/DriverTests.cs
+++ b/csharp/test/Drivers/BigQuery/DriverTests.cs
@@ -299,6 +299,36 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
}
}
+ /// <summary>
+ /// Validates if the driver can connect to a live server and
+ /// parse the results.
+ /// </summary>
+ [SkippableFact, Order(6)]
+ public void CanExecuteParallelQueries()
+ {
+ foreach (BigQueryTestEnvironment environment in _environments)
+ {
+ AdbcConnection adbcConnection =
GetAdbcConnection(environment.Name);
+
+ Parallel.For(0, environment.NumberOfParallelRuns, (i) =>
+ {
+ Parallel.ForEach(environment.ParallelQueries, pq =>
+ {
+ using (AdbcStatement statement =
adbcConnection.CreateStatement())
+ {
+ statement.SqlQuery = pq.Query;
+
+ QueryResult queryResult = statement.ExecuteQuery();
+
+ _outputHelper?.WriteLine($"({i})
{DateTime.Now.Ticks} - {queryResult.RowCount} results for {pq.Query}");
+
+ Tests.DriverTests.CanExecuteQuery(queryResult,
pq.ExpectedResultsCount, environment.Name);
+ }
+ });
+ });
+ }
+ }
+
private AdbcConnection GetAdbcConnection(string? environmentName)
{
if (string.IsNullOrEmpty(environmentName))
