CurtHagenlocher commented on code in PR #1192:
URL: https://github.com/apache/arrow-adbc/pull/1192#discussion_r1364699977
##########
csharp/src/Client/AdbcCommand.cs:
##########
@@ -167,11 +167,13 @@ public new AdbcDataReader ExecuteReader(CommandBehavior
behavior)
{
switch (behavior)
{
- case CommandBehavior.SingleRow:
- throw new InvalidOperationException("Cannot read rows in
Arrow");
- default:
+ case CommandBehavior.SchemaOnly: // The schema is not known
until a read happens
Review Comment:
I think the expectation with this behavior is that the schema is returned
but not the data, which wouldn't end up being true with this implementation. I
don't know how important that is.
SchemaOnly evaluation is supported in ADBC 1.1, FWIW, and I'm working on the
C# implementation of that.
##########
csharp/src/Drivers/BigQuery/BigQueryStatement.cs:
##########
@@ -0,0 +1,422 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using Apache.Arrow.Ipc;
+using Apache.Arrow.Types;
+using Google.Apis.Auth.OAuth2;
+using Google.Cloud.BigQuery.Storage.V1;
+using Google.Cloud.BigQuery.V2;
+using TableFieldSchema = Google.Apis.Bigquery.v2.Data.TableFieldSchema;
+using TableSchema = Google.Apis.Bigquery.v2.Data.TableSchema;
+
+namespace Apache.Arrow.Adbc.Drivers.BigQuery
+{
+ /// <summary>
+ /// BigQuery-specific implementation of <see cref="AdbcStatement"/>
+ /// </summary>
+ public class BigQueryStatement : AdbcStatement
+ {
+ readonly BigQueryClient client;
+ readonly GoogleCredential credential;
+
+ public BigQueryStatement(BigQueryClient client, GoogleCredential
credential)
+ {
+ this.client = client;
+ this.credential = credential;
+ }
+
+ public IReadOnlyDictionary<string, string>? Options { get; set; }
+
+ public override QueryResult ExecuteQuery()
+ {
+ QueryOptions? queryOptions = ValidateOptions();
+ BigQueryJob job = this.client.CreateQueryJob(SqlQuery, null,
queryOptions);
+ BigQueryResults results = job.GetQueryResults();
+
+ BigQueryReadClientBuilder readClientBuilder = new
BigQueryReadClientBuilder();
+ readClientBuilder.Credential = this.credential;
+ BigQueryReadClient readClient = readClientBuilder.Build();
+
+ // TODO: translate the schema
Review Comment:
What did you have in mind here? Is the schema not being translated at line
70?
##########
csharp/src/Drivers/BigQuery/BigQueryStatement.cs:
##########
@@ -0,0 +1,422 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using Apache.Arrow.Ipc;
+using Apache.Arrow.Types;
+using Google.Apis.Auth.OAuth2;
+using Google.Cloud.BigQuery.Storage.V1;
+using Google.Cloud.BigQuery.V2;
+using TableFieldSchema = Google.Apis.Bigquery.v2.Data.TableFieldSchema;
+using TableSchema = Google.Apis.Bigquery.v2.Data.TableSchema;
+
+namespace Apache.Arrow.Adbc.Drivers.BigQuery
+{
+ /// <summary>
+ /// BigQuery-specific implementation of <see cref="AdbcStatement"/>
+ /// </summary>
+ public class BigQueryStatement : AdbcStatement
+ {
+ readonly BigQueryClient client;
+ readonly GoogleCredential credential;
+
+ public BigQueryStatement(BigQueryClient client, GoogleCredential
credential)
+ {
+ this.client = client;
+ this.credential = credential;
+ }
+
+ public IReadOnlyDictionary<string, string>? Options { get; set; }
+
+ public override QueryResult ExecuteQuery()
+ {
+ QueryOptions? queryOptions = ValidateOptions();
+ BigQueryJob job = this.client.CreateQueryJob(SqlQuery, null,
queryOptions);
+ BigQueryResults results = job.GetQueryResults();
+
+ BigQueryReadClientBuilder readClientBuilder = new
BigQueryReadClientBuilder();
+ readClientBuilder.Credential = this.credential;
+ BigQueryReadClient readClient = readClientBuilder.Build();
+
+ // TODO: translate the schema
+
+ string table =
$"projects/{results.TableReference.ProjectId}/datasets/{results.TableReference.DatasetId}/tables/{results.TableReference.TableId}";
+
+ ReadSession rs = new ReadSession { Table = table, DataFormat =
DataFormat.Arrow };
+ ReadSession rrs = readClient.CreateReadSession("projects/" +
results.TableReference.ProjectId, rs, 1);
+
+ long totalRows = results.TotalRows == null ? -1L :
(long)results.TotalRows.Value;
+ IArrowArrayStream stream = new
MultiArrowReader(TranslateSchema(results.Schema), rrs.Streams.Select(s =>
ReadChunk(readClient, s.Name)));
+
+ return new QueryResult(totalRows, stream);
+ }
+
+ public override UpdateResult ExecuteUpdate()
+ {
+ BigQueryResults result = this.client.ExecuteQuery(SqlQuery,
parameters: null);
+ long updatedRows = result.NumDmlAffectedRows == null ? -1L :
result.NumDmlAffectedRows.Value;
+
+ return new UpdateResult(updatedRows);
+ }
+
+ public override object GetValue(IArrowArray arrowArray, Field field,
int index)
+ {
+ if (arrowArray is Int64Array)
+ {
+ return ((Int64Array)arrowArray).Values[index];
Review Comment:
This will return the wrong value when the value is null. Use GetValue for
Int64Array and DoubleArray.
##########
csharp/src/Drivers/BigQuery/BigQueryStatement.cs:
##########
@@ -0,0 +1,422 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using Apache.Arrow.Ipc;
+using Apache.Arrow.Types;
+using Google.Apis.Auth.OAuth2;
+using Google.Cloud.BigQuery.Storage.V1;
+using Google.Cloud.BigQuery.V2;
+using TableFieldSchema = Google.Apis.Bigquery.v2.Data.TableFieldSchema;
+using TableSchema = Google.Apis.Bigquery.v2.Data.TableSchema;
+
+namespace Apache.Arrow.Adbc.Drivers.BigQuery
+{
+ /// <summary>
+ /// BigQuery-specific implementation of <see cref="AdbcStatement"/>
+ /// </summary>
+ public class BigQueryStatement : AdbcStatement
+ {
+ readonly BigQueryClient client;
+ readonly GoogleCredential credential;
+
+ public BigQueryStatement(BigQueryClient client, GoogleCredential
credential)
+ {
+ this.client = client;
+ this.credential = credential;
+ }
+
+ public IReadOnlyDictionary<string, string>? Options { get; set; }
+
+ public override QueryResult ExecuteQuery()
+ {
+ QueryOptions? queryOptions = ValidateOptions();
+ BigQueryJob job = this.client.CreateQueryJob(SqlQuery, null,
queryOptions);
+ BigQueryResults results = job.GetQueryResults();
+
+ BigQueryReadClientBuilder readClientBuilder = new
BigQueryReadClientBuilder();
+ readClientBuilder.Credential = this.credential;
+ BigQueryReadClient readClient = readClientBuilder.Build();
+
+ // TODO: translate the schema
+
+ string table =
$"projects/{results.TableReference.ProjectId}/datasets/{results.TableReference.DatasetId}/tables/{results.TableReference.TableId}";
+
+ ReadSession rs = new ReadSession { Table = table, DataFormat =
DataFormat.Arrow };
+ ReadSession rrs = readClient.CreateReadSession("projects/" +
results.TableReference.ProjectId, rs, 1);
+
+ long totalRows = results.TotalRows == null ? -1L :
(long)results.TotalRows.Value;
+ IArrowArrayStream stream = new
MultiArrowReader(TranslateSchema(results.Schema), rrs.Streams.Select(s =>
ReadChunk(readClient, s.Name)));
+
+ return new QueryResult(totalRows, stream);
+ }
+
+ public override UpdateResult ExecuteUpdate()
+ {
+ BigQueryResults result = this.client.ExecuteQuery(SqlQuery,
parameters: null);
+ long updatedRows = result.NumDmlAffectedRows == null ? -1L :
result.NumDmlAffectedRows.Value;
+
+ return new UpdateResult(updatedRows);
+ }
+
+ public override object GetValue(IArrowArray arrowArray, Field field,
int index)
+ {
+ if (arrowArray is Int64Array)
+ {
+ return ((Int64Array)arrowArray).Values[index];
+ }
+ else if (arrowArray is DoubleArray)
+ {
+ return ((DoubleArray)arrowArray).Values[index];
+ }
+ else if (arrowArray is Decimal128Array)
+ {
+ try
+ {
+ // the value may be <decimal.min or >decimal.max
+ // then Arrow throws an exception
+ // no good way to check prior to
+ return ((Decimal128Array)arrowArray).GetValue(index);
+ }
+ catch (OverflowException oex)
+ {
+ return ParseDecimalValueFromOverflowException(oex);
+ }
+ }
+ else if (arrowArray is Decimal256Array)
+ {
+ try
+ {
+ return ((Decimal256Array)arrowArray).GetValue(index);
+ }
+ catch (OverflowException oex)
+ {
+ return ParseDecimalValueFromOverflowException(oex);
+ }
+ }
+ else if (arrowArray is BooleanArray)
+ {
+ return ((BooleanArray)arrowArray).GetValue(index);
+ }
+ else if (arrowArray is StringArray)
+ {
+ return ((StringArray)arrowArray).GetString(index);
+ }
+ else if (arrowArray is BinaryArray)
+ {
+ ReadOnlySpan<byte> bytes =
((BinaryArray)arrowArray).GetBytes(index);
+
+ if (bytes != null)
+ return bytes.ToArray();
+ }
+ else if (arrowArray is Date32Array)
+ {
+ Date32Array date32Array = (Date32Array)arrowArray;
+
+ return date32Array.GetDateTime(index);
+ }
+ else if (arrowArray is Date64Array)
+ {
+ Date64Array date64Array = (Date64Array)arrowArray;
+
+ return date64Array.GetDateTime(index);
+ }
+ else if (arrowArray is Time64Array)
+ {
+ return ((Time64Array)arrowArray).GetValue(index);
+ }
+ else if (arrowArray is TimestampArray)
+ {
+ TimestampArray timestampArray = (TimestampArray)arrowArray;
+ DateTimeOffset dateTimeOffset =
timestampArray.GetTimestamp(index).Value;
+ return dateTimeOffset;
+ }
+ else if (arrowArray is StructArray)
+ {
+ StructArray structArray = (StructArray)arrowArray;
+ return SerializeToJson(structArray, index);
+ }
+ // maybe not be needed?
+ else if (arrowArray is ListArray)
+ {
+ return ((ListArray)arrowArray).GetSlicedValues(index);
+ }
+
+ return null;
+ }
+
+ static Schema TranslateSchema(TableSchema schema)
+ {
+ return new Schema(schema.Fields.Select(TranslateField), null);
+ }
+
+ static Field TranslateField(TableFieldSchema field)
+ {
+ return new Field(field.Name, TranslateType(field), field.Mode ==
"NULLABLE");
+ }
+
+ static IArrowType TranslateType(TableFieldSchema field)
+ {
+ // per
https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/com/google/api/services/bigquery/model/TableFieldSchema.html#getType--
+
+ switch (field.Type)
+ {
+ case "INTEGER" or "INT64":
+ return Int64Type.Default;
+ case "FLOAT" or "FLOAT64":
+ return DoubleType.Default;
+ case "BOOL" or "BOOLEAN":
+ return BooleanType.Default;
+ case "STRING":
+ return StringType.Default;
+ case "BYTES":
+ return BinaryType.Default;
+ case "DATETIME":
+ return TimestampType.Default;
+ case "TIMESTAMP":
+ return TimestampType.Default;
+ case "TIME":
+ return Time64Type.Default;
+ case "DATE":
+ return Date64Type.Default;
+ case "RECORD" or "STRUCT":
+ // its a json string
+ return StringType.Default;
+ // get schema cannot get precison and scale
+ case "NUMERIC" or "DECIMAL":
+ return new Decimal128Type(38, 9);
+ //(int)field.Precision, (int)field.Scale);
+
+ // treat these values as strings
+ case "BIGNUMERIC" or "BIGDECIMAL" or "GEOGRAPHY":
+ return StringType.Default;
Review Comment:
Maybe to avoid having a breaking change in the future, this behavior could
be driven by an option which says "report large decimals as strings?"
##########
csharp/src/Drivers/BigQuery/BigQueryStatement.cs:
##########
@@ -0,0 +1,422 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using Apache.Arrow.Ipc;
+using Apache.Arrow.Types;
+using Google.Apis.Auth.OAuth2;
+using Google.Cloud.BigQuery.Storage.V1;
+using Google.Cloud.BigQuery.V2;
+using TableFieldSchema = Google.Apis.Bigquery.v2.Data.TableFieldSchema;
+using TableSchema = Google.Apis.Bigquery.v2.Data.TableSchema;
+
+namespace Apache.Arrow.Adbc.Drivers.BigQuery
+{
+ /// <summary>
+ /// BigQuery-specific implementation of <see cref="AdbcStatement"/>
+ /// </summary>
+ public class BigQueryStatement : AdbcStatement
+ {
+ readonly BigQueryClient client;
+ readonly GoogleCredential credential;
+
+ public BigQueryStatement(BigQueryClient client, GoogleCredential
credential)
+ {
+ this.client = client;
+ this.credential = credential;
+ }
+
+ public IReadOnlyDictionary<string, string>? Options { get; set; }
+
+ public override QueryResult ExecuteQuery()
+ {
+ QueryOptions? queryOptions = ValidateOptions();
+ BigQueryJob job = this.client.CreateQueryJob(SqlQuery, null,
queryOptions);
+ BigQueryResults results = job.GetQueryResults();
+
+ BigQueryReadClientBuilder readClientBuilder = new
BigQueryReadClientBuilder();
+ readClientBuilder.Credential = this.credential;
+ BigQueryReadClient readClient = readClientBuilder.Build();
+
+ // TODO: translate the schema
+
+ string table =
$"projects/{results.TableReference.ProjectId}/datasets/{results.TableReference.DatasetId}/tables/{results.TableReference.TableId}";
+
+ ReadSession rs = new ReadSession { Table = table, DataFormat =
DataFormat.Arrow };
+ ReadSession rrs = readClient.CreateReadSession("projects/" +
results.TableReference.ProjectId, rs, 1);
+
+ long totalRows = results.TotalRows == null ? -1L :
(long)results.TotalRows.Value;
+ IArrowArrayStream stream = new
MultiArrowReader(TranslateSchema(results.Schema), rrs.Streams.Select(s =>
ReadChunk(readClient, s.Name)));
+
+ return new QueryResult(totalRows, stream);
+ }
+
+ public override UpdateResult ExecuteUpdate()
+ {
+ BigQueryResults result = this.client.ExecuteQuery(SqlQuery,
parameters: null);
+ long updatedRows = result.NumDmlAffectedRows == null ? -1L :
result.NumDmlAffectedRows.Value;
+
+ return new UpdateResult(updatedRows);
+ }
+
+ public override object GetValue(IArrowArray arrowArray, Field field,
int index)
+ {
+ if (arrowArray is Int64Array)
+ {
+ return ((Int64Array)arrowArray).Values[index];
+ }
+ else if (arrowArray is DoubleArray)
+ {
+ return ((DoubleArray)arrowArray).Values[index];
+ }
+ else if (arrowArray is Decimal128Array)
+ {
+ try
+ {
+ // the value may be <decimal.min or >decimal.max
+ // then Arrow throws an exception
+ // no good way to check prior to
+ return ((Decimal128Array)arrowArray).GetValue(index);
+ }
+ catch (OverflowException oex)
+ {
+ return ParseDecimalValueFromOverflowException(oex);
+ }
+ }
+ else if (arrowArray is Decimal256Array)
+ {
+ try
+ {
+ return ((Decimal256Array)arrowArray).GetValue(index);
+ }
+ catch (OverflowException oex)
+ {
+ return ParseDecimalValueFromOverflowException(oex);
+ }
+ }
+ else if (arrowArray is BooleanArray)
+ {
+ return ((BooleanArray)arrowArray).GetValue(index);
+ }
+ else if (arrowArray is StringArray)
+ {
+ return ((StringArray)arrowArray).GetString(index);
+ }
+ else if (arrowArray is BinaryArray)
+ {
+ ReadOnlySpan<byte> bytes =
((BinaryArray)arrowArray).GetBytes(index);
+
+ if (bytes != null)
+ return bytes.ToArray();
+ }
+ else if (arrowArray is Date32Array)
+ {
+ Date32Array date32Array = (Date32Array)arrowArray;
+
+ return date32Array.GetDateTime(index);
+ }
+ else if (arrowArray is Date64Array)
+ {
+ Date64Array date64Array = (Date64Array)arrowArray;
+
+ return date64Array.GetDateTime(index);
+ }
+ else if (arrowArray is Time64Array)
+ {
+ return ((Time64Array)arrowArray).GetValue(index);
+ }
+ else if (arrowArray is TimestampArray)
+ {
+ TimestampArray timestampArray = (TimestampArray)arrowArray;
+ DateTimeOffset dateTimeOffset =
timestampArray.GetTimestamp(index).Value;
+ return dateTimeOffset;
+ }
+ else if (arrowArray is StructArray)
+ {
+ StructArray structArray = (StructArray)arrowArray;
+ return SerializeToJson(structArray, index);
+ }
+ // maybe not be needed?
+ else if (arrowArray is ListArray)
+ {
+ return ((ListArray)arrowArray).GetSlicedValues(index);
+ }
+
+ return null;
+ }
+
+ static Schema TranslateSchema(TableSchema schema)
+ {
+ return new Schema(schema.Fields.Select(TranslateField), null);
+ }
+
+ static Field TranslateField(TableFieldSchema field)
+ {
+ return new Field(field.Name, TranslateType(field), field.Mode ==
"NULLABLE");
+ }
+
+ static IArrowType TranslateType(TableFieldSchema field)
+ {
+ // per
https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/com/google/api/services/bigquery/model/TableFieldSchema.html#getType--
+
+ switch (field.Type)
+ {
+ case "INTEGER" or "INT64":
+ return Int64Type.Default;
+ case "FLOAT" or "FLOAT64":
+ return DoubleType.Default;
+ case "BOOL" or "BOOLEAN":
+ return BooleanType.Default;
+ case "STRING":
+ return StringType.Default;
+ case "BYTES":
+ return BinaryType.Default;
+ case "DATETIME":
+ return TimestampType.Default;
+ case "TIMESTAMP":
+ return TimestampType.Default;
+ case "TIME":
+ return Time64Type.Default;
+ case "DATE":
+ return Date64Type.Default;
+ case "RECORD" or "STRUCT":
+ // its a json string
+ return StringType.Default;
+ // get schema cannot get precison and scale
+ case "NUMERIC" or "DECIMAL":
+ return new Decimal128Type(38, 9);
Review Comment:
I guess the test works because 38, 9 are the precision and scale in the test
data?
##########
csharp/src/Drivers/BigQuery/readme.md:
##########
@@ -0,0 +1,88 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+
+# BigQuery
+The BigQuery ADBC driver wraps a
[BigQueryClient](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.BigQueryClient)
object for working with [Google BigQuery](https://cloud.google.com/bigquery/)
data.
+
+# Supported Features
+
+## Authentication
+
+The ADBC driver supports both Service and User accounts for use with [BigQuery
authentication](https://cloud.google.com/bigquery/docs/authentication/).
+
+## Authorization
+
+The ADBC driver passes the configured credentials to BigQuery, but you may
need to ensure the credentials have proper
[authorization](https://cloud.google.com/bigquery/docs/authorization/) to
perform operations such as read and write.
+
+## Parameters
+
+**adbc.bigquery.allow_large_results**<br>
+ Sets the
[AllowLargeResults](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.QueryOptions#Google_Cloud_BigQuery_V2_QueryOptions_AllowLargeResults)
value of the QueryOptions to `true` if configured; otherwise, the default is
`false`.
+
+**adbc.bigquery.auth_type**<br>
+ Required. Must be `user` or `service`
+
+https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.QueryOptions#Google_Cloud_BigQuery_V2_QueryOptions_AllowLargeResults
+
+**adbc.bigquery.client_id**<br>
+ The OAuth client ID. Required for `user`
authentication.
+
+**adbc.bigquery.client_secret**<br>
+ The OAuth client secret. Required for `user`
authentication.
+
+**adbc.bigquery.auth_json_credential**<br>
+ Required if using `service` authentication. This value
is passed to the
[GoogleCredential.FromJson](https://cloud.google.com/dotnet/docs/reference/Google.Apis/latest/Google.Apis.Auth.OAuth2.GoogleCredential#Google_Apis_Auth_OAuth2_GoogleCredential_FromJson_System_String)
method.
+
+**adbc.bigquery.project_id**<br>
+ The [Project
ID](https://cloud.google.com/resource-manager/docs/creating-managing-projects)
used for accessing BigQuery.
+
+**adbc.bigquery.refresh_token**<br>
+ The refresh token used for when the generated OAuth
token expires. Required for `user` authentication.
+
+**adbc.bigquery.use_legacy_sql**<br>
+ Sets the
[UseLegacySql](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.QueryOptions#Google_Cloud_BigQuery_V2_QueryOptions_UseLegacySql)
value of the QueryOptions to `true` if configured; otherwise, the default is
`false`.
+
+
+## Type Support
+
+There are some limitations to both C# and the C# Arrow implementation that
limit how [BigQuery data
types](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types)
that can be represented by the ADBC driver. For example, the `BIGNUMERIC` type
in BigQuery does not have a large value equivalent to C#. Also, the C# Arrow
does library does not have a
[Map](https://arrow.apache.org/docs/python/generated/pyarrow.map_.html#pyarrow.map_)
implementation.
+
Review Comment:
Information about "Map" is obsolete ;).
##########
csharp/src/Drivers/BigQuery/BigQueryStatement.cs:
##########
@@ -0,0 +1,422 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using Apache.Arrow.Ipc;
+using Apache.Arrow.Types;
+using Google.Apis.Auth.OAuth2;
+using Google.Cloud.BigQuery.Storage.V1;
+using Google.Cloud.BigQuery.V2;
+using TableFieldSchema = Google.Apis.Bigquery.v2.Data.TableFieldSchema;
+using TableSchema = Google.Apis.Bigquery.v2.Data.TableSchema;
+
+namespace Apache.Arrow.Adbc.Drivers.BigQuery
+{
+ /// <summary>
+ /// BigQuery-specific implementation of <see cref="AdbcStatement"/>
+ /// </summary>
+ public class BigQueryStatement : AdbcStatement
+ {
+ readonly BigQueryClient client;
+ readonly GoogleCredential credential;
+
+ public BigQueryStatement(BigQueryClient client, GoogleCredential
credential)
+ {
+ this.client = client;
+ this.credential = credential;
+ }
+
+ public IReadOnlyDictionary<string, string>? Options { get; set; }
+
+ public override QueryResult ExecuteQuery()
+ {
+ QueryOptions? queryOptions = ValidateOptions();
+ BigQueryJob job = this.client.CreateQueryJob(SqlQuery, null,
queryOptions);
+ BigQueryResults results = job.GetQueryResults();
+
+ BigQueryReadClientBuilder readClientBuilder = new
BigQueryReadClientBuilder();
+ readClientBuilder.Credential = this.credential;
+ BigQueryReadClient readClient = readClientBuilder.Build();
+
+ // TODO: translate the schema
+
+ string table =
$"projects/{results.TableReference.ProjectId}/datasets/{results.TableReference.DatasetId}/tables/{results.TableReference.TableId}";
+
+ ReadSession rs = new ReadSession { Table = table, DataFormat =
DataFormat.Arrow };
+ ReadSession rrs = readClient.CreateReadSession("projects/" +
results.TableReference.ProjectId, rs, 1);
+
+ long totalRows = results.TotalRows == null ? -1L :
(long)results.TotalRows.Value;
+ IArrowArrayStream stream = new
MultiArrowReader(TranslateSchema(results.Schema), rrs.Streams.Select(s =>
ReadChunk(readClient, s.Name)));
+
+ return new QueryResult(totalRows, stream);
+ }
+
+ public override UpdateResult ExecuteUpdate()
+ {
+ BigQueryResults result = this.client.ExecuteQuery(SqlQuery,
parameters: null);
+ long updatedRows = result.NumDmlAffectedRows == null ? -1L :
result.NumDmlAffectedRows.Value;
+
+ return new UpdateResult(updatedRows);
+ }
+
+ public override object GetValue(IArrowArray arrowArray, Field field,
int index)
+ {
+ if (arrowArray is Int64Array)
Review Comment:
Consider using a switch on either the type or on
arrowArray.Data.DataType.TypeId. The former would also allow casting in place
(i.e. "case Int64Array int64Array: return int64array.GetValue(index);"
##########
csharp/src/Drivers/BigQuery/BigQueryConnection.cs:
##########
@@ -0,0 +1,954 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Linq;
+using System.Net.Http;
+using System.Text;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using Apache.Arrow.Ipc;
+using Apache.Arrow.Types;
+using Google.Api.Gax;
+using Google.Apis.Auth.OAuth2;
+using Google.Apis.Bigquery.v2.Data;
+using Google.Cloud.BigQuery.V2;
+
+namespace Apache.Arrow.Adbc.Drivers.BigQuery
+{
+ /// <summary>
+ /// BigQuery-specific implementation of <see cref="AdbcConnection"/>
+ /// </summary>
+ public class BigQueryConnection : AdbcConnection
+ {
+ readonly IReadOnlyDictionary<string, string> properties;
+ BigQueryClient? client;
+ GoogleCredential? credential;
+
+ const string infoDriverName = "ADBC BigQuery Driver";
+ const string infoDriverVersion = "1.0.0";
+ const string infoVendorName = "BigQuery";
+ const string infoDriverArrowVersion = "1.0.0";
+
+ readonly IReadOnlyList<AdbcInfoCode> infoSupportedCodes = new
List<AdbcInfoCode> {
+ AdbcInfoCode.DriverName,
+ AdbcInfoCode.DriverVersion,
+ AdbcInfoCode.DriverArrowVersion,
+ AdbcInfoCode.VendorName
+ };
+
+ public BigQueryConnection(IReadOnlyDictionary<string, string>
properties)
+ {
+ this.properties = properties;
+ }
+
+ /// <summary>
+ /// Initializes the internal BigQuery connection
+ /// </summary>
+ /// <exception cref="ArgumentException"></exception>
+ internal void Open()
+ {
+ string projectId = string.Empty;
+ string clientId = string.Empty;
+ string clientSecret = string.Empty;
+ string refreshToken = string.Empty;
+
+ string tokenEndpoint = BigQueryConstants.TokenEndpoint;
+
+ string authenticationType =
BigQueryConstants.UserAuthenticationType;
+
+ // TODO: handle token expiration
+
+ if (!this.properties.TryGetValue(BigQueryParameters.ProjectId, out
projectId))
+ throw new ArgumentException($"The
{BigQueryParameters.ProjectId} parameter is not present");
+
+ if
(this.properties.ContainsKey(BigQueryParameters.AuthenticationType))
+ {
+
this.properties.TryGetValue(BigQueryParameters.AuthenticationType, out
authenticationType);
+
+
if(!authenticationType.Equals(BigQueryConstants.UserAuthenticationType,
StringComparison.OrdinalIgnoreCase) &&
+
!authenticationType.Equals(BigQueryConstants.ServiceAccountAuthenticationType,
StringComparison.OrdinalIgnoreCase))
+ {
+ throw new ArgumentException($"The
{BigQueryParameters.AuthenticationType} parameter can only be
`{BigQueryConstants.UserAuthenticationType}` or
`{BigQueryConstants.ServiceAccountAuthenticationType}`");
+ }
+ }
+
+ if
(authenticationType.Equals(BigQueryConstants.UserAuthenticationType,
StringComparison.OrdinalIgnoreCase))
+ {
+ if (!this.properties.TryGetValue(BigQueryParameters.ClientId,
out clientId))
+ throw new ArgumentException($"The
{BigQueryParameters.ClientId} parameter is not present");
+
+ if
(!this.properties.TryGetValue(BigQueryParameters.ClientSecret, out
clientSecret))
+ throw new ArgumentException($"The
{BigQueryParameters.ClientSecret} parameter is not present");
+
+ if
(!this.properties.TryGetValue(BigQueryParameters.RefreshToken, out
refreshToken))
+ throw new ArgumentException($"The
{BigQueryParameters.RefreshToken} parameter is not present");
+
+ this.credential =
GoogleCredential.FromAccessToken(GetAccessToken(clientId, clientSecret,
refreshToken, tokenEndpoint));
+ }
+ else
+ {
+ string json = string.Empty;
+
+ if
(!this.properties.TryGetValue(BigQueryParameters.JsonCredential, out json))
+ throw new ArgumentException($"The
{BigQueryParameters.JsonCredential} parameter is not present");
+
+ this.credential = GoogleCredential.FromJson(json);
+ }
+
+ this.client = BigQueryClient.Create(projectId, this.credential);
+ }
+
+ public override IArrowArrayStream GetInfo(List<AdbcInfoCode> codes)
+ {
+ const int strValTypeID = 0;
+
+ UnionType infoUnionType = new UnionType(
+ new List<Field>()
+ {
+ new Field("string_value", StringType.Default, true),
+ new Field("bool_value", BooleanType.Default, true),
+ new Field("int64_value", Int64Type.Default, true),
+ new Field("int32_bitmask", Int32Type.Default, true),
+ new Field(
+ "string_list",
+ new ListType(
+ new Field("item", StringType.Default, true)
+ ),
+ false
+ ),
+ new Field(
+ "int32_to_int32_list_map",
+ new ListType(
+ new Field("entries", new StructType(
+ new List<Field>()
+ {
+ new Field("key", Int32Type.Default, false),
+ new Field("value", Int32Type.Default,
true),
+ }
+ ), false)
+ ),
+ true
+ )
+ },
+ new int[] { 0, 1, 2, 3, 4, 5 }.ToArray(),
+ UnionMode.Dense);
+
+ if (codes.Count == 0)
+ {
+ codes = new List<AdbcInfoCode>(infoSupportedCodes);
+ }
+
+ UInt32Array.Builder infoNameBuilder = new UInt32Array.Builder();
+ ArrowBuffer.Builder<byte> typeBuilder = new
ArrowBuffer.Builder<byte>();
+ ArrowBuffer.Builder<int> offsetBuilder = new
ArrowBuffer.Builder<int>();
+ StringArray.Builder stringInfoBuilder = new StringArray.Builder();
+ int nullCount = 0;
+ int arrayLength = codes.Count;
+
+ foreach (AdbcInfoCode code in codes)
+ {
+ switch (code)
+ {
+ case AdbcInfoCode.DriverName:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.Append(infoDriverName);
+ break;
+ case AdbcInfoCode.DriverVersion:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.Append(infoDriverVersion);
+ break;
+ case AdbcInfoCode.DriverArrowVersion:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.Append(infoDriverArrowVersion);
+ break;
+ case AdbcInfoCode.VendorName:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.Append(infoVendorName);
+ break;
+ default:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.AppendNull();
+ nullCount++;
+ break;
+ }
+ }
+
+ StructType entryType = new StructType(
+ new List<Field>(){
+ new Field("key", Int32Type.Default, false),
+ new Field("value", Int32Type.Default, true)});
+
+ StructArray entriesDataArray = new StructArray(entryType, 0,
+ new[] { new Int32Array.Builder().Build(), new
Int32Array.Builder().Build() },
+ new ArrowBuffer.BitmapBuilder().Build());
+
+ List<IArrowArray> childrenArrays = new List<IArrowArray>()
+ {
+ stringInfoBuilder.Build(),
+ new BooleanArray.Builder().Build(),
+ new Int64Array.Builder().Build(),
+ new Int32Array.Builder().Build(),
+ new ListArray.Builder(StringType.Default).Build(),
+ CreateNestedListArray(new List<IArrowArray?>(){
entriesDataArray }, entryType)
+ };
+
+ DenseUnionArray infoValue = new DenseUnionArray(infoUnionType,
arrayLength, childrenArrays, typeBuilder.Build(), offsetBuilder.Build(),
nullCount);
+
+ List<IArrowArray> dataArrays = new List<IArrowArray>
+ {
+ infoNameBuilder.Build(),
+ infoValue
+ };
+
+ return new BigQueryInfoArrowStream(StandardSchemas.GetInfoSchema,
dataArrays, 4);
+ }
+
+ public override IArrowArrayStream GetObjects(
+ GetObjectsDepth depth,
+ string catalogPattern,
+ string dbSchemaPattern,
+ string tableNamePattern,
+ List<string> tableTypes,
+ string columnNamePattern)
+ {
+ List<IArrowArray> dataArrays = GetCatalogs(depth, catalogPattern,
dbSchemaPattern,
+ tableNamePattern, tableTypes, columnNamePattern);
+
+ return new
BigQueryInfoArrowStream(StandardSchemas.GetObjectsSchema, dataArrays, 1);
+ }
+
+ private List<IArrowArray> GetCatalogs(
+ GetObjectsDepth depth,
+ string catalogPattern,
+ string dbSchemaPattern,
+ string tableNamePattern,
+ List<string> tableTypes,
+ string columnNamePattern)
+ {
+ StringArray.Builder catalogNameBuilder = new StringArray.Builder();
+ List<IArrowArray?> catalogDbSchemasValues = new
List<IArrowArray?>();
+ string catalogRegexp = PatternToRegEx(catalogPattern);
+ PagedEnumerable<ProjectList, CloudProject> catalogs =
this.client.ListProjects();
+
+ foreach (CloudProject catalog in catalogs)
+ {
+ if (Regex.IsMatch(catalog.ProjectId, catalogRegexp,
RegexOptions.IgnoreCase))
+ {
+ catalogNameBuilder.Append(catalog.ProjectId);
+
+ if (depth == GetObjectsDepth.Catalogs)
+ {
+ catalogDbSchemasValues.Add(null);
+ }
+ else
+ {
+ catalogDbSchemasValues.Add(GetDbSchemas(
+ depth, catalog.ProjectId, dbSchemaPattern,
+ tableNamePattern, tableTypes, columnNamePattern));
+ }
+ }
+ }
+
+ List<IArrowArray> dataArrays = new List<IArrowArray>
+ {
+ catalogNameBuilder.Build(),
+ CreateNestedListArray(catalogDbSchemasValues, new
StructType(StandardSchemas.DbSchemaSchema)),
+ };
+
+ return dataArrays;
+ }
+
+ private StructArray GetDbSchemas(
+ GetObjectsDepth depth,
+ string catalog,
+ string dbSchemaPattern,
+ string tableNamePattern,
+ List<string> tableTypes,
+ string columnNamePattern)
+ {
+ StringArray.Builder dbSchemaNameBuilder = new
StringArray.Builder();
+ List<IArrowArray?> dbSchemaTablesValues = new List<IArrowArray?>();
+ ArrowBuffer.BitmapBuilder nullBitmapBuffer = new
ArrowBuffer.BitmapBuilder();
+ int length = 0;
+
+ string dbSchemaRegexp = PatternToRegEx(dbSchemaPattern);
+
+ PagedEnumerable<DatasetList, BigQueryDataset> schemas =
this.client.ListDatasets(catalog);
+
+ foreach (BigQueryDataset schema in schemas)
+ {
+ if (Regex.IsMatch(schema.Reference.DatasetId, dbSchemaRegexp,
RegexOptions.IgnoreCase))
+ {
+ dbSchemaNameBuilder.Append(schema.Reference.DatasetId);
+ length++;
+ nullBitmapBuffer.Append(true);
+
+ if (depth == GetObjectsDepth.DbSchemas)
+ {
+ dbSchemaTablesValues.Add(null);
+ }
+ else
+ {
+ dbSchemaTablesValues.Add(GetTableSchemas(
+ depth, catalog, schema.Reference.DatasetId,
+ tableNamePattern, tableTypes, columnNamePattern));
+ }
+ }
+ }
+
+ List<IArrowArray> dataArrays = new List<IArrowArray>
+ {
+ dbSchemaNameBuilder.Build(),
+ CreateNestedListArray(dbSchemaTablesValues, new
StructType(StandardSchemas.TableSchema)),
+ };
+
+ return new StructArray(
+ new StructType(StandardSchemas.DbSchemaSchema),
+ length,
+ dataArrays,
+ nullBitmapBuffer.Build());
+ }
+
+ private StructArray GetTableSchemas(
+ GetObjectsDepth depth,
+ string catalog,
+ string dbSchema,
+ string tableNamePattern,
+ List<string> tableTypes,
+ string columnNamePattern)
+ {
+ StringArray.Builder tableNameBuilder = new StringArray.Builder();
+ StringArray.Builder tableTypeBuilder = new StringArray.Builder();
+ List<IArrowArray?> tableColumnsValues = new List<IArrowArray?>();
+ List<IArrowArray?> tableConstraintsValues = new
List<IArrowArray?>();
+ ArrowBuffer.BitmapBuilder nullBitmapBuffer = new
ArrowBuffer.BitmapBuilder();
+ int length = 0;
+
+ string query = string.Format("SELECT * FROM
`{0}`.`{1}`.INFORMATION_SCHEMA.TABLES",
+ catalog, dbSchema);
+
+ if (tableNamePattern != null)
+ {
+ query = string.Concat(query, string.Format(" WHERE table_name
LIKE '{0}'", tableNamePattern));
+ if (tableTypes.Count > 0)
+ {
+ query = string.Concat(query, string.Format(" AND
table_type IN ('{0}')", string.Join("', '", tableTypes).ToUpper()));
Review Comment:
tableTypes comes from the caller, making this susceptible to SQL injection.
##########
csharp/src/Drivers/BigQuery/BigQueryStatement.cs:
##########
@@ -0,0 +1,422 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using Apache.Arrow.Ipc;
+using Apache.Arrow.Types;
+using Google.Apis.Auth.OAuth2;
+using Google.Cloud.BigQuery.Storage.V1;
+using Google.Cloud.BigQuery.V2;
+using TableFieldSchema = Google.Apis.Bigquery.v2.Data.TableFieldSchema;
+using TableSchema = Google.Apis.Bigquery.v2.Data.TableSchema;
+
+namespace Apache.Arrow.Adbc.Drivers.BigQuery
+{
+ /// <summary>
+ /// BigQuery-specific implementation of <see cref="AdbcStatement"/>
+ /// </summary>
+ public class BigQueryStatement : AdbcStatement
+ {
+ readonly BigQueryClient client;
+ readonly GoogleCredential credential;
+
+ public BigQueryStatement(BigQueryClient client, GoogleCredential
credential)
+ {
+ this.client = client;
+ this.credential = credential;
+ }
+
+ public IReadOnlyDictionary<string, string>? Options { get; set; }
+
+ public override QueryResult ExecuteQuery()
+ {
+ QueryOptions? queryOptions = ValidateOptions();
+ BigQueryJob job = this.client.CreateQueryJob(SqlQuery, null,
queryOptions);
+ BigQueryResults results = job.GetQueryResults();
+
+ BigQueryReadClientBuilder readClientBuilder = new
BigQueryReadClientBuilder();
+ readClientBuilder.Credential = this.credential;
+ BigQueryReadClient readClient = readClientBuilder.Build();
+
+ // TODO: translate the schema
+
+ string table =
$"projects/{results.TableReference.ProjectId}/datasets/{results.TableReference.DatasetId}/tables/{results.TableReference.TableId}";
+
+ ReadSession rs = new ReadSession { Table = table, DataFormat =
DataFormat.Arrow };
+ ReadSession rrs = readClient.CreateReadSession("projects/" +
results.TableReference.ProjectId, rs, 1);
+
+ long totalRows = results.TotalRows == null ? -1L :
(long)results.TotalRows.Value;
+ IArrowArrayStream stream = new
MultiArrowReader(TranslateSchema(results.Schema), rrs.Streams.Select(s =>
ReadChunk(readClient, s.Name)));
+
+ return new QueryResult(totalRows, stream);
+ }
+
+ public override UpdateResult ExecuteUpdate()
+ {
+ BigQueryResults result = this.client.ExecuteQuery(SqlQuery,
parameters: null);
+ long updatedRows = result.NumDmlAffectedRows == null ? -1L :
result.NumDmlAffectedRows.Value;
+
+ return new UpdateResult(updatedRows);
+ }
+
+ public override object GetValue(IArrowArray arrowArray, Field field,
int index)
+ {
+ if (arrowArray is Int64Array)
+ {
+ return ((Int64Array)arrowArray).Values[index];
+ }
+ else if (arrowArray is DoubleArray)
+ {
+ return ((DoubleArray)arrowArray).Values[index];
+ }
+ else if (arrowArray is Decimal128Array)
+ {
+ try
+ {
+ // the value may be <decimal.min or >decimal.max
+ // then Arrow throws an exception
+ // no good way to check prior to
+ return ((Decimal128Array)arrowArray).GetValue(index);
+ }
+ catch (OverflowException oex)
+ {
+ return ParseDecimalValueFromOverflowException(oex);
+ }
+ }
+ else if (arrowArray is Decimal256Array)
+ {
+ try
+ {
+ return ((Decimal256Array)arrowArray).GetValue(index);
+ }
+ catch (OverflowException oex)
+ {
+ return ParseDecimalValueFromOverflowException(oex);
+ }
+ }
+ else if (arrowArray is BooleanArray)
+ {
+ return ((BooleanArray)arrowArray).GetValue(index);
+ }
+ else if (arrowArray is StringArray)
+ {
+ return ((StringArray)arrowArray).GetString(index);
+ }
+ else if (arrowArray is BinaryArray)
+ {
+ ReadOnlySpan<byte> bytes =
((BinaryArray)arrowArray).GetBytes(index);
+
+ if (bytes != null)
+ return bytes.ToArray();
+ }
+ else if (arrowArray is Date32Array)
+ {
+ Date32Array date32Array = (Date32Array)arrowArray;
+
+ return date32Array.GetDateTime(index);
+ }
+ else if (arrowArray is Date64Array)
+ {
+ Date64Array date64Array = (Date64Array)arrowArray;
+
+ return date64Array.GetDateTime(index);
+ }
+ else if (arrowArray is Time64Array)
+ {
+ return ((Time64Array)arrowArray).GetValue(index);
+ }
+ else if (arrowArray is TimestampArray)
+ {
+ TimestampArray timestampArray = (TimestampArray)arrowArray;
+ DateTimeOffset dateTimeOffset =
timestampArray.GetTimestamp(index).Value;
+ return dateTimeOffset;
+ }
+ else if (arrowArray is StructArray)
+ {
+ StructArray structArray = (StructArray)arrowArray;
+ return SerializeToJson(structArray, index);
+ }
+ // maybe not be needed?
+ else if (arrowArray is ListArray)
+ {
+ return ((ListArray)arrowArray).GetSlicedValues(index);
+ }
+
+ return null;
+ }
+
+ static Schema TranslateSchema(TableSchema schema)
+ {
+ return new Schema(schema.Fields.Select(TranslateField), null);
+ }
+
+ static Field TranslateField(TableFieldSchema field)
+ {
+ return new Field(field.Name, TranslateType(field), field.Mode ==
"NULLABLE");
+ }
+
+ static IArrowType TranslateType(TableFieldSchema field)
+ {
+ // per
https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/com/google/api/services/bigquery/model/TableFieldSchema.html#getType--
+
+ switch (field.Type)
+ {
+ case "INTEGER" or "INT64":
+ return Int64Type.Default;
+ case "FLOAT" or "FLOAT64":
+ return DoubleType.Default;
+ case "BOOL" or "BOOLEAN":
+ return BooleanType.Default;
+ case "STRING":
+ return StringType.Default;
+ case "BYTES":
+ return BinaryType.Default;
+ case "DATETIME":
+ return TimestampType.Default;
+ case "TIMESTAMP":
+ return TimestampType.Default;
+ case "TIME":
+ return Time64Type.Default;
+ case "DATE":
+ return Date64Type.Default;
+ case "RECORD" or "STRUCT":
+ // its a json string
+ return StringType.Default;
+ // get schema cannot get precison and scale
+ case "NUMERIC" or "DECIMAL":
+ return new Decimal128Type(38, 9);
Review Comment:
If we don't know the precision and scale of the column, the data can't get
interpreted correctly. Are they always 38 and 9 or are these basically
placeholders until we can produce the correct value?
##########
csharp/src/Drivers/BigQuery/BigQueryConnection.cs:
##########
@@ -0,0 +1,954 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Linq;
+using System.Net.Http;
+using System.Text;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using Apache.Arrow.Ipc;
+using Apache.Arrow.Types;
+using Google.Api.Gax;
+using Google.Apis.Auth.OAuth2;
+using Google.Apis.Bigquery.v2.Data;
+using Google.Cloud.BigQuery.V2;
+
+namespace Apache.Arrow.Adbc.Drivers.BigQuery
+{
+ /// <summary>
+ /// BigQuery-specific implementation of <see cref="AdbcConnection"/>
+ /// </summary>
+ public class BigQueryConnection : AdbcConnection
+ {
+ readonly IReadOnlyDictionary<string, string> properties;
+ BigQueryClient? client;
+ GoogleCredential? credential;
+
+ const string infoDriverName = "ADBC BigQuery Driver";
+ const string infoDriverVersion = "1.0.0";
+ const string infoVendorName = "BigQuery";
+ const string infoDriverArrowVersion = "1.0.0";
+
+ readonly IReadOnlyList<AdbcInfoCode> infoSupportedCodes = new
List<AdbcInfoCode> {
+ AdbcInfoCode.DriverName,
+ AdbcInfoCode.DriverVersion,
+ AdbcInfoCode.DriverArrowVersion,
+ AdbcInfoCode.VendorName
+ };
+
+ public BigQueryConnection(IReadOnlyDictionary<string, string>
properties)
+ {
+ this.properties = properties;
+ }
+
+ /// <summary>
+ /// Initializes the internal BigQuery connection
+ /// </summary>
+ /// <exception cref="ArgumentException"></exception>
+ internal void Open()
+ {
+ string projectId = string.Empty;
+ string clientId = string.Empty;
+ string clientSecret = string.Empty;
+ string refreshToken = string.Empty;
+
+ string tokenEndpoint = BigQueryConstants.TokenEndpoint;
+
+ string authenticationType =
BigQueryConstants.UserAuthenticationType;
+
+ // TODO: handle token expiration
+
+ if (!this.properties.TryGetValue(BigQueryParameters.ProjectId, out
projectId))
+ throw new ArgumentException($"The
{BigQueryParameters.ProjectId} parameter is not present");
+
+ if
(this.properties.ContainsKey(BigQueryParameters.AuthenticationType))
+ {
+
this.properties.TryGetValue(BigQueryParameters.AuthenticationType, out
authenticationType);
+
+
if(!authenticationType.Equals(BigQueryConstants.UserAuthenticationType,
StringComparison.OrdinalIgnoreCase) &&
+
!authenticationType.Equals(BigQueryConstants.ServiceAccountAuthenticationType,
StringComparison.OrdinalIgnoreCase))
+ {
+ throw new ArgumentException($"The
{BigQueryParameters.AuthenticationType} parameter can only be
`{BigQueryConstants.UserAuthenticationType}` or
`{BigQueryConstants.ServiceAccountAuthenticationType}`");
+ }
+ }
+
+ if
(authenticationType.Equals(BigQueryConstants.UserAuthenticationType,
StringComparison.OrdinalIgnoreCase))
+ {
+ if (!this.properties.TryGetValue(BigQueryParameters.ClientId,
out clientId))
+ throw new ArgumentException($"The
{BigQueryParameters.ClientId} parameter is not present");
+
+ if
(!this.properties.TryGetValue(BigQueryParameters.ClientSecret, out
clientSecret))
+ throw new ArgumentException($"The
{BigQueryParameters.ClientSecret} parameter is not present");
+
+ if
(!this.properties.TryGetValue(BigQueryParameters.RefreshToken, out
refreshToken))
+ throw new ArgumentException($"The
{BigQueryParameters.RefreshToken} parameter is not present");
+
+ this.credential =
GoogleCredential.FromAccessToken(GetAccessToken(clientId, clientSecret,
refreshToken, tokenEndpoint));
+ }
+ else
+ {
+ string json = string.Empty;
+
+ if
(!this.properties.TryGetValue(BigQueryParameters.JsonCredential, out json))
+ throw new ArgumentException($"The
{BigQueryParameters.JsonCredential} parameter is not present");
+
+ this.credential = GoogleCredential.FromJson(json);
+ }
+
+ this.client = BigQueryClient.Create(projectId, this.credential);
+ }
+
+ public override IArrowArrayStream GetInfo(List<AdbcInfoCode> codes)
+ {
+ const int strValTypeID = 0;
+
+ UnionType infoUnionType = new UnionType(
+ new List<Field>()
+ {
+ new Field("string_value", StringType.Default, true),
+ new Field("bool_value", BooleanType.Default, true),
+ new Field("int64_value", Int64Type.Default, true),
+ new Field("int32_bitmask", Int32Type.Default, true),
+ new Field(
+ "string_list",
+ new ListType(
+ new Field("item", StringType.Default, true)
+ ),
+ false
+ ),
+ new Field(
+ "int32_to_int32_list_map",
+ new ListType(
+ new Field("entries", new StructType(
+ new List<Field>()
+ {
+ new Field("key", Int32Type.Default, false),
+ new Field("value", Int32Type.Default,
true),
+ }
+ ), false)
+ ),
+ true
+ )
+ },
+ new int[] { 0, 1, 2, 3, 4, 5 }.ToArray(),
+ UnionMode.Dense);
+
+ if (codes.Count == 0)
+ {
+ codes = new List<AdbcInfoCode>(infoSupportedCodes);
+ }
+
+ UInt32Array.Builder infoNameBuilder = new UInt32Array.Builder();
+ ArrowBuffer.Builder<byte> typeBuilder = new
ArrowBuffer.Builder<byte>();
+ ArrowBuffer.Builder<int> offsetBuilder = new
ArrowBuffer.Builder<int>();
+ StringArray.Builder stringInfoBuilder = new StringArray.Builder();
+ int nullCount = 0;
+ int arrayLength = codes.Count;
+
+ foreach (AdbcInfoCode code in codes)
+ {
+ switch (code)
+ {
+ case AdbcInfoCode.DriverName:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.Append(infoDriverName);
+ break;
+ case AdbcInfoCode.DriverVersion:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.Append(infoDriverVersion);
+ break;
+ case AdbcInfoCode.DriverArrowVersion:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.Append(infoDriverArrowVersion);
+ break;
+ case AdbcInfoCode.VendorName:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.Append(infoVendorName);
+ break;
+ default:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.AppendNull();
+ nullCount++;
+ break;
+ }
+ }
+
+ StructType entryType = new StructType(
+ new List<Field>(){
+ new Field("key", Int32Type.Default, false),
+ new Field("value", Int32Type.Default, true)});
+
+ StructArray entriesDataArray = new StructArray(entryType, 0,
+ new[] { new Int32Array.Builder().Build(), new
Int32Array.Builder().Build() },
+ new ArrowBuffer.BitmapBuilder().Build());
+
+ List<IArrowArray> childrenArrays = new List<IArrowArray>()
+ {
+ stringInfoBuilder.Build(),
+ new BooleanArray.Builder().Build(),
+ new Int64Array.Builder().Build(),
+ new Int32Array.Builder().Build(),
+ new ListArray.Builder(StringType.Default).Build(),
+ CreateNestedListArray(new List<IArrowArray?>(){
entriesDataArray }, entryType)
+ };
+
+ DenseUnionArray infoValue = new DenseUnionArray(infoUnionType,
arrayLength, childrenArrays, typeBuilder.Build(), offsetBuilder.Build(),
nullCount);
+
+ List<IArrowArray> dataArrays = new List<IArrowArray>
+ {
+ infoNameBuilder.Build(),
+ infoValue
+ };
+
+ return new BigQueryInfoArrowStream(StandardSchemas.GetInfoSchema,
dataArrays, 4);
+ }
+
+ public override IArrowArrayStream GetObjects(
+ GetObjectsDepth depth,
+ string catalogPattern,
+ string dbSchemaPattern,
+ string tableNamePattern,
+ List<string> tableTypes,
+ string columnNamePattern)
+ {
+ List<IArrowArray> dataArrays = GetCatalogs(depth, catalogPattern,
dbSchemaPattern,
+ tableNamePattern, tableTypes, columnNamePattern);
+
+ return new
BigQueryInfoArrowStream(StandardSchemas.GetObjectsSchema, dataArrays, 1);
+ }
+
+ private List<IArrowArray> GetCatalogs(
+ GetObjectsDepth depth,
+ string catalogPattern,
+ string dbSchemaPattern,
+ string tableNamePattern,
+ List<string> tableTypes,
+ string columnNamePattern)
+ {
+ StringArray.Builder catalogNameBuilder = new StringArray.Builder();
+ List<IArrowArray?> catalogDbSchemasValues = new
List<IArrowArray?>();
+ string catalogRegexp = PatternToRegEx(catalogPattern);
+ PagedEnumerable<ProjectList, CloudProject> catalogs =
this.client.ListProjects();
+
+ foreach (CloudProject catalog in catalogs)
+ {
+ if (Regex.IsMatch(catalog.ProjectId, catalogRegexp,
RegexOptions.IgnoreCase))
+ {
+ catalogNameBuilder.Append(catalog.ProjectId);
+
+ if (depth == GetObjectsDepth.Catalogs)
+ {
+ catalogDbSchemasValues.Add(null);
+ }
+ else
+ {
+ catalogDbSchemasValues.Add(GetDbSchemas(
+ depth, catalog.ProjectId, dbSchemaPattern,
+ tableNamePattern, tableTypes, columnNamePattern));
+ }
+ }
+ }
+
+ List<IArrowArray> dataArrays = new List<IArrowArray>
+ {
+ catalogNameBuilder.Build(),
+ CreateNestedListArray(catalogDbSchemasValues, new
StructType(StandardSchemas.DbSchemaSchema)),
+ };
+
+ return dataArrays;
+ }
+
+ private StructArray GetDbSchemas(
+ GetObjectsDepth depth,
+ string catalog,
+ string dbSchemaPattern,
+ string tableNamePattern,
+ List<string> tableTypes,
+ string columnNamePattern)
+ {
+ StringArray.Builder dbSchemaNameBuilder = new
StringArray.Builder();
+ List<IArrowArray?> dbSchemaTablesValues = new List<IArrowArray?>();
+ ArrowBuffer.BitmapBuilder nullBitmapBuffer = new
ArrowBuffer.BitmapBuilder();
+ int length = 0;
+
+ string dbSchemaRegexp = PatternToRegEx(dbSchemaPattern);
+
+ PagedEnumerable<DatasetList, BigQueryDataset> schemas =
this.client.ListDatasets(catalog);
+
+ foreach (BigQueryDataset schema in schemas)
+ {
+ if (Regex.IsMatch(schema.Reference.DatasetId, dbSchemaRegexp,
RegexOptions.IgnoreCase))
+ {
+ dbSchemaNameBuilder.Append(schema.Reference.DatasetId);
+ length++;
+ nullBitmapBuffer.Append(true);
+
+ if (depth == GetObjectsDepth.DbSchemas)
+ {
+ dbSchemaTablesValues.Add(null);
+ }
+ else
+ {
+ dbSchemaTablesValues.Add(GetTableSchemas(
+ depth, catalog, schema.Reference.DatasetId,
+ tableNamePattern, tableTypes, columnNamePattern));
+ }
+ }
+ }
+
+ List<IArrowArray> dataArrays = new List<IArrowArray>
+ {
+ dbSchemaNameBuilder.Build(),
+ CreateNestedListArray(dbSchemaTablesValues, new
StructType(StandardSchemas.TableSchema)),
+ };
+
+ return new StructArray(
+ new StructType(StandardSchemas.DbSchemaSchema),
+ length,
+ dataArrays,
+ nullBitmapBuffer.Build());
+ }
+
+ private StructArray GetTableSchemas(
+ GetObjectsDepth depth,
+ string catalog,
+ string dbSchema,
+ string tableNamePattern,
+ List<string> tableTypes,
+ string columnNamePattern)
+ {
+ StringArray.Builder tableNameBuilder = new StringArray.Builder();
+ StringArray.Builder tableTypeBuilder = new StringArray.Builder();
+ List<IArrowArray?> tableColumnsValues = new List<IArrowArray?>();
+ List<IArrowArray?> tableConstraintsValues = new
List<IArrowArray?>();
+ ArrowBuffer.BitmapBuilder nullBitmapBuffer = new
ArrowBuffer.BitmapBuilder();
+ int length = 0;
+
+ string query = string.Format("SELECT * FROM
`{0}`.`{1}`.INFORMATION_SCHEMA.TABLES",
+ catalog, dbSchema);
+
+ if (tableNamePattern != null)
+ {
+ query = string.Concat(query, string.Format(" WHERE table_name
LIKE '{0}'", tableNamePattern));
Review Comment:
tableNamePattern comes from the caller, making this susceptible to SQL
injection
##########
csharp/src/Drivers/BigQuery/readme.md:
##########
@@ -0,0 +1,88 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+
+# BigQuery
+The BigQuery ADBC driver wraps a
[BigQueryClient](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.BigQueryClient)
object for working with [Google BigQuery](https://cloud.google.com/bigquery/)
data.
+
+# Supported Features
+
+## Authentication
+
+The ADBC driver supports both Service and User accounts for use with [BigQuery
authentication](https://cloud.google.com/bigquery/docs/authentication/).
+
+## Authorization
+
+The ADBC driver passes the configured credentials to BigQuery, but you may
need to ensure the credentials have proper
[authorization](https://cloud.google.com/bigquery/docs/authorization/) to
perform operations such as read and write.
+
+## Parameters
+
+**adbc.bigquery.allow_large_results**<br>
+ Sets the
[AllowLargeResults](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.QueryOptions#Google_Cloud_BigQuery_V2_QueryOptions_AllowLargeResults)
value of the QueryOptions to `true` if configured; otherwise, the default is
`false`.
+
+**adbc.bigquery.auth_type**<br>
+ Required. Must be `user` or `service`
+
+https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.QueryOptions#Google_Cloud_BigQuery_V2_QueryOptions_AllowLargeResults
+
+**adbc.bigquery.client_id**<br>
+ The OAuth client ID. Required for `user`
authentication.
+
+**adbc.bigquery.client_secret**<br>
+ The OAuth client secret. Required for `user`
authentication.
+
+**adbc.bigquery.auth_json_credential**<br>
+ Required if using `service` authentication. This value
is passed to the
[GoogleCredential.FromJson](https://cloud.google.com/dotnet/docs/reference/Google.Apis/latest/Google.Apis.Auth.OAuth2.GoogleCredential#Google_Apis_Auth_OAuth2_GoogleCredential_FromJson_System_String)
method.
+
+**adbc.bigquery.project_id**<br>
+ The [Project
ID](https://cloud.google.com/resource-manager/docs/creating-managing-projects)
used for accessing BigQuery.
+
+**adbc.bigquery.refresh_token**<br>
+ The refresh token used for when the generated OAuth
token expires. Required for `user` authentication.
+
+**adbc.bigquery.use_legacy_sql**<br>
+ Sets the
[UseLegacySql](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.QueryOptions#Google_Cloud_BigQuery_V2_QueryOptions_UseLegacySql)
value of the QueryOptions to `true` if configured; otherwise, the default is
`false`.
+
+
+## Type Support
+
+There are some limitations to both C# and the C# Arrow implementation that
limit how [BigQuery data
types](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types)
that can be represented by the ADBC driver. For example, the `BIGNUMERIC` type
in BigQuery does not have a large value equivalent to C#. Also, the C# Arrow
does library does not have a
[Map](https://arrow.apache.org/docs/python/generated/pyarrow.map_.html#pyarrow.map_)
implementation.
+
+The following table depicts how the BigQuery ADBC driver converts a BigQuery
type to an Arrow type.
+
+| BigQuery Type | Arrow Type | C# Type
+|----------|:-------------:|
+| BIGNUMERIC | Decimal256 | decimal / string*
+| BOOL | Boolean | bool
+| BYTES | Binary | byte[]
+| DATE | Date64 | DateTime
+| DATETIME | Timestamp | DateTime
+| FLOAT64 | Double | double
+| GEOGRAPHY | String | string
+| INT64 | Int64 | long
+| NUMERIC | Decimal128 | decimal / string*
+| STRING | String | string
+| STRUCT | String+ | string
+| TIME |Time64 | long
+| TIMESTAMP | Timestamp | DateTimeOffset
+
+*An attempt is made to parse the original value as a `decimal` in C#. If that
fails, the driver attempts to parse the overflow exception and return the
original value.
Review Comment:
We obviously need a better way to handle decimal values that are too large
for the BCL decimal type, but that will probably need to happen at the Arrow
layer first.
##########
csharp/src/Drivers/BigQuery/BigQueryConnection.cs:
##########
@@ -0,0 +1,954 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Linq;
+using System.Net.Http;
+using System.Text;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using Apache.Arrow.Ipc;
+using Apache.Arrow.Types;
+using Google.Api.Gax;
+using Google.Apis.Auth.OAuth2;
+using Google.Apis.Bigquery.v2.Data;
+using Google.Cloud.BigQuery.V2;
+
+namespace Apache.Arrow.Adbc.Drivers.BigQuery
+{
+ /// <summary>
+ /// BigQuery-specific implementation of <see cref="AdbcConnection"/>
+ /// </summary>
+ public class BigQueryConnection : AdbcConnection
+ {
+ readonly IReadOnlyDictionary<string, string> properties;
+ BigQueryClient? client;
+ GoogleCredential? credential;
+
+ const string infoDriverName = "ADBC BigQuery Driver";
+ const string infoDriverVersion = "1.0.0";
+ const string infoVendorName = "BigQuery";
+ const string infoDriverArrowVersion = "1.0.0";
+
+ readonly IReadOnlyList<AdbcInfoCode> infoSupportedCodes = new
List<AdbcInfoCode> {
+ AdbcInfoCode.DriverName,
+ AdbcInfoCode.DriverVersion,
+ AdbcInfoCode.DriverArrowVersion,
+ AdbcInfoCode.VendorName
+ };
+
+ public BigQueryConnection(IReadOnlyDictionary<string, string>
properties)
+ {
+ this.properties = properties;
+ }
+
+ /// <summary>
+ /// Initializes the internal BigQuery connection
+ /// </summary>
+ /// <exception cref="ArgumentException"></exception>
+ internal void Open()
+ {
+ string projectId = string.Empty;
+ string clientId = string.Empty;
+ string clientSecret = string.Empty;
+ string refreshToken = string.Empty;
+
+ string tokenEndpoint = BigQueryConstants.TokenEndpoint;
+
+ string authenticationType =
BigQueryConstants.UserAuthenticationType;
+
+ // TODO: handle token expiration
+
+ if (!this.properties.TryGetValue(BigQueryParameters.ProjectId, out
projectId))
+ throw new ArgumentException($"The
{BigQueryParameters.ProjectId} parameter is not present");
+
+ if
(this.properties.ContainsKey(BigQueryParameters.AuthenticationType))
+ {
+
this.properties.TryGetValue(BigQueryParameters.AuthenticationType, out
authenticationType);
+
+
if(!authenticationType.Equals(BigQueryConstants.UserAuthenticationType,
StringComparison.OrdinalIgnoreCase) &&
+
!authenticationType.Equals(BigQueryConstants.ServiceAccountAuthenticationType,
StringComparison.OrdinalIgnoreCase))
+ {
+ throw new ArgumentException($"The
{BigQueryParameters.AuthenticationType} parameter can only be
`{BigQueryConstants.UserAuthenticationType}` or
`{BigQueryConstants.ServiceAccountAuthenticationType}`");
+ }
+ }
+
+ if
(authenticationType.Equals(BigQueryConstants.UserAuthenticationType,
StringComparison.OrdinalIgnoreCase))
+ {
+ if (!this.properties.TryGetValue(BigQueryParameters.ClientId,
out clientId))
+ throw new ArgumentException($"The
{BigQueryParameters.ClientId} parameter is not present");
+
+ if
(!this.properties.TryGetValue(BigQueryParameters.ClientSecret, out
clientSecret))
+ throw new ArgumentException($"The
{BigQueryParameters.ClientSecret} parameter is not present");
+
+ if
(!this.properties.TryGetValue(BigQueryParameters.RefreshToken, out
refreshToken))
+ throw new ArgumentException($"The
{BigQueryParameters.RefreshToken} parameter is not present");
+
+ this.credential =
GoogleCredential.FromAccessToken(GetAccessToken(clientId, clientSecret,
refreshToken, tokenEndpoint));
+ }
+ else
+ {
+ string json = string.Empty;
+
+ if
(!this.properties.TryGetValue(BigQueryParameters.JsonCredential, out json))
+ throw new ArgumentException($"The
{BigQueryParameters.JsonCredential} parameter is not present");
+
+ this.credential = GoogleCredential.FromJson(json);
+ }
+
+ this.client = BigQueryClient.Create(projectId, this.credential);
+ }
+
+ public override IArrowArrayStream GetInfo(List<AdbcInfoCode> codes)
+ {
+ const int strValTypeID = 0;
+
+ UnionType infoUnionType = new UnionType(
+ new List<Field>()
+ {
+ new Field("string_value", StringType.Default, true),
+ new Field("bool_value", BooleanType.Default, true),
+ new Field("int64_value", Int64Type.Default, true),
+ new Field("int32_bitmask", Int32Type.Default, true),
+ new Field(
+ "string_list",
+ new ListType(
+ new Field("item", StringType.Default, true)
+ ),
+ false
+ ),
+ new Field(
+ "int32_to_int32_list_map",
+ new ListType(
+ new Field("entries", new StructType(
+ new List<Field>()
+ {
+ new Field("key", Int32Type.Default, false),
+ new Field("value", Int32Type.Default,
true),
+ }
+ ), false)
+ ),
+ true
+ )
+ },
+ new int[] { 0, 1, 2, 3, 4, 5 }.ToArray(),
+ UnionMode.Dense);
+
+ if (codes.Count == 0)
+ {
+ codes = new List<AdbcInfoCode>(infoSupportedCodes);
+ }
+
+ UInt32Array.Builder infoNameBuilder = new UInt32Array.Builder();
+ ArrowBuffer.Builder<byte> typeBuilder = new
ArrowBuffer.Builder<byte>();
+ ArrowBuffer.Builder<int> offsetBuilder = new
ArrowBuffer.Builder<int>();
+ StringArray.Builder stringInfoBuilder = new StringArray.Builder();
+ int nullCount = 0;
+ int arrayLength = codes.Count;
+
+ foreach (AdbcInfoCode code in codes)
+ {
+ switch (code)
+ {
+ case AdbcInfoCode.DriverName:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.Append(infoDriverName);
+ break;
+ case AdbcInfoCode.DriverVersion:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.Append(infoDriverVersion);
+ break;
+ case AdbcInfoCode.DriverArrowVersion:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.Append(infoDriverArrowVersion);
+ break;
+ case AdbcInfoCode.VendorName:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.Append(infoVendorName);
+ break;
+ default:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.AppendNull();
+ nullCount++;
+ break;
+ }
+ }
+
+ StructType entryType = new StructType(
+ new List<Field>(){
+ new Field("key", Int32Type.Default, false),
+ new Field("value", Int32Type.Default, true)});
+
+ StructArray entriesDataArray = new StructArray(entryType, 0,
+ new[] { new Int32Array.Builder().Build(), new
Int32Array.Builder().Build() },
+ new ArrowBuffer.BitmapBuilder().Build());
+
+ List<IArrowArray> childrenArrays = new List<IArrowArray>()
+ {
+ stringInfoBuilder.Build(),
+ new BooleanArray.Builder().Build(),
+ new Int64Array.Builder().Build(),
+ new Int32Array.Builder().Build(),
+ new ListArray.Builder(StringType.Default).Build(),
+ CreateNestedListArray(new List<IArrowArray?>(){
entriesDataArray }, entryType)
+ };
+
+ DenseUnionArray infoValue = new DenseUnionArray(infoUnionType,
arrayLength, childrenArrays, typeBuilder.Build(), offsetBuilder.Build(),
nullCount);
+
+ List<IArrowArray> dataArrays = new List<IArrowArray>
+ {
+ infoNameBuilder.Build(),
+ infoValue
+ };
+
+ return new BigQueryInfoArrowStream(StandardSchemas.GetInfoSchema,
dataArrays, 4);
+ }
+
+ public override IArrowArrayStream GetObjects(
+ GetObjectsDepth depth,
+ string catalogPattern,
+ string dbSchemaPattern,
+ string tableNamePattern,
+ List<string> tableTypes,
+ string columnNamePattern)
+ {
+ List<IArrowArray> dataArrays = GetCatalogs(depth, catalogPattern,
dbSchemaPattern,
+ tableNamePattern, tableTypes, columnNamePattern);
+
+ return new
BigQueryInfoArrowStream(StandardSchemas.GetObjectsSchema, dataArrays, 1);
+ }
+
+ private List<IArrowArray> GetCatalogs(
+ GetObjectsDepth depth,
+ string catalogPattern,
+ string dbSchemaPattern,
+ string tableNamePattern,
+ List<string> tableTypes,
+ string columnNamePattern)
+ {
+ StringArray.Builder catalogNameBuilder = new StringArray.Builder();
+ List<IArrowArray?> catalogDbSchemasValues = new
List<IArrowArray?>();
+ string catalogRegexp = PatternToRegEx(catalogPattern);
+ PagedEnumerable<ProjectList, CloudProject> catalogs =
this.client.ListProjects();
+
+ foreach (CloudProject catalog in catalogs)
+ {
+ if (Regex.IsMatch(catalog.ProjectId, catalogRegexp,
RegexOptions.IgnoreCase))
+ {
+ catalogNameBuilder.Append(catalog.ProjectId);
+
+ if (depth == GetObjectsDepth.Catalogs)
+ {
+ catalogDbSchemasValues.Add(null);
+ }
+ else
+ {
+ catalogDbSchemasValues.Add(GetDbSchemas(
+ depth, catalog.ProjectId, dbSchemaPattern,
+ tableNamePattern, tableTypes, columnNamePattern));
+ }
+ }
+ }
+
+ List<IArrowArray> dataArrays = new List<IArrowArray>
+ {
+ catalogNameBuilder.Build(),
+ CreateNestedListArray(catalogDbSchemasValues, new
StructType(StandardSchemas.DbSchemaSchema)),
+ };
+
+ return dataArrays;
+ }
+
+ private StructArray GetDbSchemas(
+ GetObjectsDepth depth,
+ string catalog,
+ string dbSchemaPattern,
+ string tableNamePattern,
+ List<string> tableTypes,
+ string columnNamePattern)
+ {
+ StringArray.Builder dbSchemaNameBuilder = new
StringArray.Builder();
+ List<IArrowArray?> dbSchemaTablesValues = new List<IArrowArray?>();
+ ArrowBuffer.BitmapBuilder nullBitmapBuffer = new
ArrowBuffer.BitmapBuilder();
+ int length = 0;
+
+ string dbSchemaRegexp = PatternToRegEx(dbSchemaPattern);
+
+ PagedEnumerable<DatasetList, BigQueryDataset> schemas =
this.client.ListDatasets(catalog);
+
+ foreach (BigQueryDataset schema in schemas)
+ {
+ if (Regex.IsMatch(schema.Reference.DatasetId, dbSchemaRegexp,
RegexOptions.IgnoreCase))
+ {
+ dbSchemaNameBuilder.Append(schema.Reference.DatasetId);
+ length++;
+ nullBitmapBuffer.Append(true);
+
+ if (depth == GetObjectsDepth.DbSchemas)
+ {
+ dbSchemaTablesValues.Add(null);
+ }
+ else
+ {
+ dbSchemaTablesValues.Add(GetTableSchemas(
+ depth, catalog, schema.Reference.DatasetId,
+ tableNamePattern, tableTypes, columnNamePattern));
+ }
+ }
+ }
+
+ List<IArrowArray> dataArrays = new List<IArrowArray>
+ {
+ dbSchemaNameBuilder.Build(),
+ CreateNestedListArray(dbSchemaTablesValues, new
StructType(StandardSchemas.TableSchema)),
+ };
+
+ return new StructArray(
+ new StructType(StandardSchemas.DbSchemaSchema),
+ length,
+ dataArrays,
+ nullBitmapBuffer.Build());
+ }
+
+ private StructArray GetTableSchemas(
+ GetObjectsDepth depth,
+ string catalog,
+ string dbSchema,
+ string tableNamePattern,
+ List<string> tableTypes,
+ string columnNamePattern)
+ {
+ StringArray.Builder tableNameBuilder = new StringArray.Builder();
+ StringArray.Builder tableTypeBuilder = new StringArray.Builder();
+ List<IArrowArray?> tableColumnsValues = new List<IArrowArray?>();
+ List<IArrowArray?> tableConstraintsValues = new
List<IArrowArray?>();
+ ArrowBuffer.BitmapBuilder nullBitmapBuffer = new
ArrowBuffer.BitmapBuilder();
+ int length = 0;
+
+ string query = string.Format("SELECT * FROM
`{0}`.`{1}`.INFORMATION_SCHEMA.TABLES",
+ catalog, dbSchema);
+
+ if (tableNamePattern != null)
+ {
+ query = string.Concat(query, string.Format(" WHERE table_name
LIKE '{0}'", tableNamePattern));
+ if (tableTypes.Count > 0)
+ {
+ query = string.Concat(query, string.Format(" AND
table_type IN ('{0}')", string.Join("', '", tableTypes).ToUpper()));
+ }
+ }
+ else
+ {
+ if (tableTypes.Count > 0)
+ {
+ query = string.Concat(query, string.Format(" WHERE
table_type IN ('{0}')", string.Join("', '", tableTypes).ToUpper()));
+ }
+ }
+
+ BigQueryResults result = this.client.ExecuteQuery(query,
parameters: null);
+
+ foreach (BigQueryRow row in result)
+ {
+ tableNameBuilder.Append(row["table_name"].ToString());
+ tableTypeBuilder.Append(row["table_type"].ToString());
+ nullBitmapBuffer.Append(true);
+ length++;
+
+ tableConstraintsValues.Add(GetConstraintSchema(
+ depth, catalog, dbSchema, row["table_name"].ToString(),
columnNamePattern));
+
+ // TODO: add constraints
+ if (depth == GetObjectsDepth.Tables)
+ {
+ tableColumnsValues.Add(null);
+ }
+ else
+ {
+ tableColumnsValues.Add(GetColumnSchema(catalog, dbSchema,
row["table_name"].ToString(), columnNamePattern));
+ }
+ }
+
+ List<IArrowArray> dataArrays = new List<IArrowArray>
+ {
+ tableNameBuilder.Build(),
+ tableTypeBuilder.Build(),
+ CreateNestedListArray(tableColumnsValues, new
StructType(StandardSchemas.ColumnSchema)),
+ CreateNestedListArray(tableConstraintsValues, new
StructType(StandardSchemas.ConstraintSchema))
+ };
+
+ return new StructArray(
+ new StructType(StandardSchemas.TableSchema),
+ length,
+ dataArrays,
+ nullBitmapBuffer.Build());
+ }
+
+ private StructArray GetColumnSchema(
+ string catalog,
+ string dbSchema,
+ string table,
+ string columnNamePattern)
+ {
+ StringArray.Builder columnNameBuilder = new StringArray.Builder();
+ Int32Array.Builder ordinalPositionBuilder = new
Int32Array.Builder();
+ StringArray.Builder remarksBuilder = new StringArray.Builder();
+ Int16Array.Builder xdbcDataTypeBuilder = new Int16Array.Builder();
+ StringArray.Builder xdbcTypeNameBuilder = new
StringArray.Builder();
+ Int32Array.Builder xdbcColumnSizeBuilder = new
Int32Array.Builder();
+ Int16Array.Builder xdbcDecimalDigitsBuilder = new
Int16Array.Builder();
+ Int16Array.Builder xdbcNumPrecRadixBuilder = new
Int16Array.Builder();
+ Int16Array.Builder xdbcNullableBuilder = new Int16Array.Builder();
+ StringArray.Builder xdbcColumnDefBuilder = new
StringArray.Builder();
+ Int16Array.Builder xdbcSqlDataTypeBuilder = new
Int16Array.Builder();
+ Int16Array.Builder xdbcDatetimeSubBuilder = new
Int16Array.Builder();
+ Int32Array.Builder xdbcCharOctetLengthBuilder = new
Int32Array.Builder();
+ StringArray.Builder xdbcIsNullableBuilder = new
StringArray.Builder();
+ StringArray.Builder xdbcScopeCatalogBuilder = new
StringArray.Builder();
+ StringArray.Builder xdbcScopeSchemaBuilder = new
StringArray.Builder();
+ StringArray.Builder xdbcScopeTableBuilder = new
StringArray.Builder();
+ BooleanArray.Builder xdbcIsAutoincrementBuilder = new
BooleanArray.Builder();
+ BooleanArray.Builder xdbcIsGeneratedcolumnBuilder = new
BooleanArray.Builder();
+ ArrowBuffer.BitmapBuilder nullBitmapBuffer = new
ArrowBuffer.BitmapBuilder();
+ int length = 0;
+
+ string query = string.Format("SELECT * FROM
`{0}`.`{1}`.INFORMATION_SCHEMA.COLUMNS WHERE table_name = '{2}'",
Review Comment:
There are still places where the concatenated string come from the calling
code, and those will require something to avoid SQL injection.
##########
csharp/src/Drivers/BigQuery/BigQueryConnection.cs:
##########
@@ -0,0 +1,954 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Linq;
+using System.Net.Http;
+using System.Text;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using Apache.Arrow.Ipc;
+using Apache.Arrow.Types;
+using Google.Api.Gax;
+using Google.Apis.Auth.OAuth2;
+using Google.Apis.Bigquery.v2.Data;
+using Google.Cloud.BigQuery.V2;
+
+namespace Apache.Arrow.Adbc.Drivers.BigQuery
+{
+ /// <summary>
+ /// BigQuery-specific implementation of <see cref="AdbcConnection"/>
+ /// </summary>
+ public class BigQueryConnection : AdbcConnection
+ {
+ readonly IReadOnlyDictionary<string, string> properties;
+ BigQueryClient? client;
+ GoogleCredential? credential;
+
+ const string infoDriverName = "ADBC BigQuery Driver";
+ const string infoDriverVersion = "1.0.0";
+ const string infoVendorName = "BigQuery";
+ const string infoDriverArrowVersion = "1.0.0";
+
+ readonly IReadOnlyList<AdbcInfoCode> infoSupportedCodes = new
List<AdbcInfoCode> {
+ AdbcInfoCode.DriverName,
+ AdbcInfoCode.DriverVersion,
+ AdbcInfoCode.DriverArrowVersion,
+ AdbcInfoCode.VendorName
+ };
+
+ public BigQueryConnection(IReadOnlyDictionary<string, string>
properties)
+ {
+ this.properties = properties;
+ }
+
+ /// <summary>
+ /// Initializes the internal BigQuery connection
+ /// </summary>
+ /// <exception cref="ArgumentException"></exception>
+ internal void Open()
+ {
+ string projectId = string.Empty;
+ string clientId = string.Empty;
+ string clientSecret = string.Empty;
+ string refreshToken = string.Empty;
+
+ string tokenEndpoint = BigQueryConstants.TokenEndpoint;
+
+ string authenticationType =
BigQueryConstants.UserAuthenticationType;
+
+ // TODO: handle token expiration
+
+ if (!this.properties.TryGetValue(BigQueryParameters.ProjectId, out
projectId))
+ throw new ArgumentException($"The
{BigQueryParameters.ProjectId} parameter is not present");
+
+ if
(this.properties.ContainsKey(BigQueryParameters.AuthenticationType))
+ {
+
this.properties.TryGetValue(BigQueryParameters.AuthenticationType, out
authenticationType);
+
+
if(!authenticationType.Equals(BigQueryConstants.UserAuthenticationType,
StringComparison.OrdinalIgnoreCase) &&
+
!authenticationType.Equals(BigQueryConstants.ServiceAccountAuthenticationType,
StringComparison.OrdinalIgnoreCase))
+ {
+ throw new ArgumentException($"The
{BigQueryParameters.AuthenticationType} parameter can only be
`{BigQueryConstants.UserAuthenticationType}` or
`{BigQueryConstants.ServiceAccountAuthenticationType}`");
+ }
+ }
+
+ if
(authenticationType.Equals(BigQueryConstants.UserAuthenticationType,
StringComparison.OrdinalIgnoreCase))
+ {
+ if (!this.properties.TryGetValue(BigQueryParameters.ClientId,
out clientId))
+ throw new ArgumentException($"The
{BigQueryParameters.ClientId} parameter is not present");
+
+ if
(!this.properties.TryGetValue(BigQueryParameters.ClientSecret, out
clientSecret))
+ throw new ArgumentException($"The
{BigQueryParameters.ClientSecret} parameter is not present");
+
+ if
(!this.properties.TryGetValue(BigQueryParameters.RefreshToken, out
refreshToken))
+ throw new ArgumentException($"The
{BigQueryParameters.RefreshToken} parameter is not present");
+
+ this.credential =
GoogleCredential.FromAccessToken(GetAccessToken(clientId, clientSecret,
refreshToken, tokenEndpoint));
+ }
+ else
+ {
+ string json = string.Empty;
+
+ if
(!this.properties.TryGetValue(BigQueryParameters.JsonCredential, out json))
+ throw new ArgumentException($"The
{BigQueryParameters.JsonCredential} parameter is not present");
+
+ this.credential = GoogleCredential.FromJson(json);
+ }
+
+ this.client = BigQueryClient.Create(projectId, this.credential);
+ }
+
+ public override IArrowArrayStream GetInfo(List<AdbcInfoCode> codes)
+ {
+ const int strValTypeID = 0;
+
+ UnionType infoUnionType = new UnionType(
+ new List<Field>()
+ {
+ new Field("string_value", StringType.Default, true),
+ new Field("bool_value", BooleanType.Default, true),
+ new Field("int64_value", Int64Type.Default, true),
+ new Field("int32_bitmask", Int32Type.Default, true),
+ new Field(
+ "string_list",
+ new ListType(
+ new Field("item", StringType.Default, true)
+ ),
+ false
+ ),
+ new Field(
+ "int32_to_int32_list_map",
+ new ListType(
+ new Field("entries", new StructType(
+ new List<Field>()
+ {
+ new Field("key", Int32Type.Default, false),
+ new Field("value", Int32Type.Default,
true),
+ }
+ ), false)
+ ),
+ true
+ )
+ },
+ new int[] { 0, 1, 2, 3, 4, 5 }.ToArray(),
+ UnionMode.Dense);
+
+ if (codes.Count == 0)
+ {
+ codes = new List<AdbcInfoCode>(infoSupportedCodes);
+ }
+
+ UInt32Array.Builder infoNameBuilder = new UInt32Array.Builder();
+ ArrowBuffer.Builder<byte> typeBuilder = new
ArrowBuffer.Builder<byte>();
+ ArrowBuffer.Builder<int> offsetBuilder = new
ArrowBuffer.Builder<int>();
+ StringArray.Builder stringInfoBuilder = new StringArray.Builder();
+ int nullCount = 0;
+ int arrayLength = codes.Count;
+
+ foreach (AdbcInfoCode code in codes)
+ {
+ switch (code)
+ {
+ case AdbcInfoCode.DriverName:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.Append(infoDriverName);
+ break;
+ case AdbcInfoCode.DriverVersion:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.Append(infoDriverVersion);
+ break;
+ case AdbcInfoCode.DriverArrowVersion:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.Append(infoDriverArrowVersion);
+ break;
+ case AdbcInfoCode.VendorName:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.Append(infoVendorName);
+ break;
+ default:
+ infoNameBuilder.Append((UInt32)code);
+ typeBuilder.Append(strValTypeID);
+ offsetBuilder.Append(stringInfoBuilder.Length);
+ stringInfoBuilder.AppendNull();
+ nullCount++;
+ break;
+ }
+ }
+
+ StructType entryType = new StructType(
+ new List<Field>(){
+ new Field("key", Int32Type.Default, false),
+ new Field("value", Int32Type.Default, true)});
+
+ StructArray entriesDataArray = new StructArray(entryType, 0,
+ new[] { new Int32Array.Builder().Build(), new
Int32Array.Builder().Build() },
+ new ArrowBuffer.BitmapBuilder().Build());
+
+ List<IArrowArray> childrenArrays = new List<IArrowArray>()
+ {
+ stringInfoBuilder.Build(),
+ new BooleanArray.Builder().Build(),
+ new Int64Array.Builder().Build(),
+ new Int32Array.Builder().Build(),
+ new ListArray.Builder(StringType.Default).Build(),
+ CreateNestedListArray(new List<IArrowArray?>(){
entriesDataArray }, entryType)
+ };
+
+ DenseUnionArray infoValue = new DenseUnionArray(infoUnionType,
arrayLength, childrenArrays, typeBuilder.Build(), offsetBuilder.Build(),
nullCount);
+
+ List<IArrowArray> dataArrays = new List<IArrowArray>
+ {
+ infoNameBuilder.Build(),
+ infoValue
+ };
+
+ return new BigQueryInfoArrowStream(StandardSchemas.GetInfoSchema,
dataArrays, 4);
+ }
+
+ public override IArrowArrayStream GetObjects(
+ GetObjectsDepth depth,
+ string catalogPattern,
+ string dbSchemaPattern,
+ string tableNamePattern,
+ List<string> tableTypes,
+ string columnNamePattern)
+ {
+ List<IArrowArray> dataArrays = GetCatalogs(depth, catalogPattern,
dbSchemaPattern,
+ tableNamePattern, tableTypes, columnNamePattern);
+
+ return new
BigQueryInfoArrowStream(StandardSchemas.GetObjectsSchema, dataArrays, 1);
+ }
+
+ private List<IArrowArray> GetCatalogs(
+ GetObjectsDepth depth,
+ string catalogPattern,
+ string dbSchemaPattern,
+ string tableNamePattern,
+ List<string> tableTypes,
+ string columnNamePattern)
+ {
+ StringArray.Builder catalogNameBuilder = new StringArray.Builder();
+ List<IArrowArray?> catalogDbSchemasValues = new
List<IArrowArray?>();
+ string catalogRegexp = PatternToRegEx(catalogPattern);
+ PagedEnumerable<ProjectList, CloudProject> catalogs =
this.client.ListProjects();
+
+ foreach (CloudProject catalog in catalogs)
+ {
+ if (Regex.IsMatch(catalog.ProjectId, catalogRegexp,
RegexOptions.IgnoreCase))
+ {
+ catalogNameBuilder.Append(catalog.ProjectId);
+
+ if (depth == GetObjectsDepth.Catalogs)
+ {
+ catalogDbSchemasValues.Add(null);
+ }
+ else
+ {
+ catalogDbSchemasValues.Add(GetDbSchemas(
+ depth, catalog.ProjectId, dbSchemaPattern,
+ tableNamePattern, tableTypes, columnNamePattern));
+ }
+ }
+ }
+
+ List<IArrowArray> dataArrays = new List<IArrowArray>
+ {
+ catalogNameBuilder.Build(),
+ CreateNestedListArray(catalogDbSchemasValues, new
StructType(StandardSchemas.DbSchemaSchema)),
+ };
+
+ return dataArrays;
+ }
+
+ private StructArray GetDbSchemas(
+ GetObjectsDepth depth,
+ string catalog,
+ string dbSchemaPattern,
+ string tableNamePattern,
+ List<string> tableTypes,
+ string columnNamePattern)
+ {
+ StringArray.Builder dbSchemaNameBuilder = new
StringArray.Builder();
+ List<IArrowArray?> dbSchemaTablesValues = new List<IArrowArray?>();
+ ArrowBuffer.BitmapBuilder nullBitmapBuffer = new
ArrowBuffer.BitmapBuilder();
+ int length = 0;
+
+ string dbSchemaRegexp = PatternToRegEx(dbSchemaPattern);
+
+ PagedEnumerable<DatasetList, BigQueryDataset> schemas =
this.client.ListDatasets(catalog);
+
+ foreach (BigQueryDataset schema in schemas)
+ {
+ if (Regex.IsMatch(schema.Reference.DatasetId, dbSchemaRegexp,
RegexOptions.IgnoreCase))
+ {
+ dbSchemaNameBuilder.Append(schema.Reference.DatasetId);
+ length++;
+ nullBitmapBuffer.Append(true);
+
+ if (depth == GetObjectsDepth.DbSchemas)
+ {
+ dbSchemaTablesValues.Add(null);
+ }
+ else
+ {
+ dbSchemaTablesValues.Add(GetTableSchemas(
+ depth, catalog, schema.Reference.DatasetId,
+ tableNamePattern, tableTypes, columnNamePattern));
+ }
+ }
+ }
+
+ List<IArrowArray> dataArrays = new List<IArrowArray>
+ {
+ dbSchemaNameBuilder.Build(),
+ CreateNestedListArray(dbSchemaTablesValues, new
StructType(StandardSchemas.TableSchema)),
+ };
+
+ return new StructArray(
+ new StructType(StandardSchemas.DbSchemaSchema),
+ length,
+ dataArrays,
+ nullBitmapBuffer.Build());
+ }
+
+ private StructArray GetTableSchemas(
+ GetObjectsDepth depth,
+ string catalog,
+ string dbSchema,
+ string tableNamePattern,
+ List<string> tableTypes,
+ string columnNamePattern)
+ {
+ StringArray.Builder tableNameBuilder = new StringArray.Builder();
+ StringArray.Builder tableTypeBuilder = new StringArray.Builder();
+ List<IArrowArray?> tableColumnsValues = new List<IArrowArray?>();
+ List<IArrowArray?> tableConstraintsValues = new
List<IArrowArray?>();
+ ArrowBuffer.BitmapBuilder nullBitmapBuffer = new
ArrowBuffer.BitmapBuilder();
+ int length = 0;
+
+ string query = string.Format("SELECT * FROM
`{0}`.`{1}`.INFORMATION_SCHEMA.TABLES",
+ catalog, dbSchema);
+
+ if (tableNamePattern != null)
+ {
+ query = string.Concat(query, string.Format(" WHERE table_name
LIKE '{0}'", tableNamePattern));
+ if (tableTypes.Count > 0)
+ {
+ query = string.Concat(query, string.Format(" AND
table_type IN ('{0}')", string.Join("', '", tableTypes).ToUpper()));
+ }
+ }
+ else
+ {
+ if (tableTypes.Count > 0)
+ {
+ query = string.Concat(query, string.Format(" WHERE
table_type IN ('{0}')", string.Join("', '", tableTypes).ToUpper()));
+ }
+ }
+
+ BigQueryResults result = this.client.ExecuteQuery(query,
parameters: null);
+
+ foreach (BigQueryRow row in result)
+ {
+ tableNameBuilder.Append(row["table_name"].ToString());
+ tableTypeBuilder.Append(row["table_type"].ToString());
+ nullBitmapBuffer.Append(true);
+ length++;
+
+ tableConstraintsValues.Add(GetConstraintSchema(
+ depth, catalog, dbSchema, row["table_name"].ToString(),
columnNamePattern));
+
+ // TODO: add constraints
+ if (depth == GetObjectsDepth.Tables)
+ {
+ tableColumnsValues.Add(null);
+ }
+ else
+ {
+ tableColumnsValues.Add(GetColumnSchema(catalog, dbSchema,
row["table_name"].ToString(), columnNamePattern));
+ }
+ }
+
+ List<IArrowArray> dataArrays = new List<IArrowArray>
+ {
+ tableNameBuilder.Build(),
+ tableTypeBuilder.Build(),
+ CreateNestedListArray(tableColumnsValues, new
StructType(StandardSchemas.ColumnSchema)),
+ CreateNestedListArray(tableConstraintsValues, new
StructType(StandardSchemas.ConstraintSchema))
+ };
+
+ return new StructArray(
+ new StructType(StandardSchemas.TableSchema),
+ length,
+ dataArrays,
+ nullBitmapBuffer.Build());
+ }
+
+ private StructArray GetColumnSchema(
+ string catalog,
+ string dbSchema,
+ string table,
+ string columnNamePattern)
+ {
+ StringArray.Builder columnNameBuilder = new StringArray.Builder();
+ Int32Array.Builder ordinalPositionBuilder = new
Int32Array.Builder();
+ StringArray.Builder remarksBuilder = new StringArray.Builder();
+ Int16Array.Builder xdbcDataTypeBuilder = new Int16Array.Builder();
+ StringArray.Builder xdbcTypeNameBuilder = new
StringArray.Builder();
+ Int32Array.Builder xdbcColumnSizeBuilder = new
Int32Array.Builder();
+ Int16Array.Builder xdbcDecimalDigitsBuilder = new
Int16Array.Builder();
+ Int16Array.Builder xdbcNumPrecRadixBuilder = new
Int16Array.Builder();
+ Int16Array.Builder xdbcNullableBuilder = new Int16Array.Builder();
+ StringArray.Builder xdbcColumnDefBuilder = new
StringArray.Builder();
+ Int16Array.Builder xdbcSqlDataTypeBuilder = new
Int16Array.Builder();
+ Int16Array.Builder xdbcDatetimeSubBuilder = new
Int16Array.Builder();
+ Int32Array.Builder xdbcCharOctetLengthBuilder = new
Int32Array.Builder();
+ StringArray.Builder xdbcIsNullableBuilder = new
StringArray.Builder();
+ StringArray.Builder xdbcScopeCatalogBuilder = new
StringArray.Builder();
+ StringArray.Builder xdbcScopeSchemaBuilder = new
StringArray.Builder();
+ StringArray.Builder xdbcScopeTableBuilder = new
StringArray.Builder();
+ BooleanArray.Builder xdbcIsAutoincrementBuilder = new
BooleanArray.Builder();
+ BooleanArray.Builder xdbcIsGeneratedcolumnBuilder = new
BooleanArray.Builder();
+ ArrowBuffer.BitmapBuilder nullBitmapBuffer = new
ArrowBuffer.BitmapBuilder();
+ int length = 0;
+
+ string query = string.Format("SELECT * FROM
`{0}`.`{1}`.INFORMATION_SCHEMA.COLUMNS WHERE table_name = '{2}'",
+ catalog, dbSchema, table);
+
+ if (columnNamePattern != null)
+ {
+ query = string.Concat(query, string.Format("AND column_name
LIKE '{0}'", columnNamePattern));
Review Comment:
columnNamePattern comes from the caller, making this susceptible to SQL
injection
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
