This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 89f36c16 feat: added override of http client as well as readme on
utils.WithAw… (#981)
89f36c16 is described below
commit 89f36c164c3e1cd65c7b3857b16464d779b0d522
Author: Ankit Sharma <[email protected]>
AuthorDate: Sun May 10 23:15:03 2026 +0530
feat: added override of http client as well as readme on utils.WithAw…
(#981)
This pr intends to solve issue:
(https://github.com/apache/iceberg-go/issues/958)
-> overriding of the HTTP client when no configuration is provided
-> as well as adding a comment on the glue NewCatalog function
---
catalog/glue/glue.go | 11 +++++++++++
io/gocloud/s3.go | 15 +++++++++++++++
2 files changed, 26 insertions(+)
diff --git a/catalog/glue/glue.go b/catalog/glue/glue.go
index 2167651a..0611aef3 100644
--- a/catalog/glue/glue.go
+++ b/catalog/glue/glue.go
@@ -145,6 +145,17 @@ type Catalog struct {
}
// NewCatalog creates a new instance of glue.Catalog with the given options.
+// To override the AWS config below is an example:
+//
+// import awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+//
+// client := awshttp.NewBuildableClient().WithTransportOptions(func(t
*http.Transport) {
+// t.IdleConnTimeout = 90 * time.Second
+// ...other transport options...
+// })
+// awsCfg.HTTPClient = client
+// catalog := glue.NewCatalog(glue.WithAwsConfig(awsCfg))
+
func NewCatalog(opts ...Option) *Catalog {
glueOps := &options{}
diff --git a/io/gocloud/s3.go b/io/gocloud/s3.go
index ea979d5d..309513fc 100644
--- a/io/gocloud/s3.go
+++ b/io/gocloud/s3.go
@@ -26,6 +26,7 @@ import (
"os"
"slices"
"strconv"
+ "time"
"github.com/apache/iceberg-go/io"
"github.com/apache/iceberg-go/utils"
@@ -131,6 +132,20 @@ func createS3Bucket(ctx context.Context, parsed *url.URL,
props map[string]strin
}
}
+ // Default HTTP client when not configured: use the SDK buildable
client so
+ // proxy, TLS, dial, and HTTP/2 behavior match the usual AWS defaults,
but
+ // raise per-host idle limits (Go's DefaultTransport uses 2 per host).
+ if awscfg.HTTPClient == nil {
+ awscfg.HTTPClient =
awshttp.NewBuildableClient().WithTransportOptions(
+ func(t *http.Transport) {
+ t.MaxIdleConns = 256
+ t.MaxIdleConnsPerHost = 256
+ t.MaxConnsPerHost = 256
+ t.IdleConnTimeout = 90 * time.Second
+ },
+ )
+ }
+
endpoint, ok := props[io.S3EndpointURL]
if !ok {
endpoint = os.Getenv("AWS_S3_ENDPOINT")
