[
https://issues.apache.org/jira/browse/BEAM-3893?focusedWorklogId=82537&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-82537
]
ASF GitHub Bot logged work on BEAM-3893:
----------------------------------------
Author: ASF GitHub Bot
Created on: 20/Mar/18 23:13
Start Date: 20/Mar/18 23:13
Worklog Time Spent: 10m
Work Description: lukecwik closed pull request #4911: [BEAM-3893] Add
fallback to unauthenticated access for Go GCS IO
URL: https://github.com/apache/beam/pull/4911
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/sdks/go/pkg/beam/io/textio/gcs/gcs.go
b/sdks/go/pkg/beam/io/textio/gcs/gcs.go
index 4a005dd7520..a5a6edfd5cd 100644
--- a/sdks/go/pkg/beam/io/textio/gcs/gcs.go
+++ b/sdks/go/pkg/beam/io/textio/gcs/gcs.go
@@ -24,6 +24,7 @@ import (
"strings"
"github.com/apache/beam/sdks/go/pkg/beam/io/textio"
+ "github.com/apache/beam/sdks/go/pkg/beam/log"
"github.com/apache/beam/sdks/go/pkg/beam/util/gcsx"
"google.golang.org/api/storage/v1"
)
@@ -37,11 +38,17 @@ type fs struct {
}
// New creates a new Google Cloud Storage filesystem using application
-// default credentials.
+// default credentials. If it fails, it falls back to unauthenticated
+// access.
func New(ctx context.Context) textio.FileSystem {
client, err := gcsx.NewClient(ctx, storage.DevstorageReadWriteScope)
if err != nil {
- panic(fmt.Sprintf("failed to create GCE client: %v", err))
+ log.Warnf(ctx, "Warning: falling back to unauthenticated GCS
access: %v", err)
+
+ client, err = gcsx.NewUnauthenticatedClient(ctx)
+ if err != nil {
+ panic(fmt.Sprintf("failed to create GCE client: %v",
err))
+ }
}
return &fs{client: client}
}
diff --git a/sdks/go/pkg/beam/provision/provision.go
b/sdks/go/pkg/beam/provision/provision.go
index 656b3f27539..7349ef2a343 100644
--- a/sdks/go/pkg/beam/provision/provision.go
+++ b/sdks/go/pkg/beam/provision/provision.go
@@ -45,7 +45,7 @@ func Info(ctx context.Context, endpoint string)
(*pb.ProvisionInfo, error) {
return nil, fmt.Errorf("failed to get manifest: %v", err)
}
if resp.GetInfo() == nil {
- return nil, fmt.Errorf("empty manifest",)
+ return nil, fmt.Errorf("empty manifest")
}
return resp.GetInfo(), nil
}
diff --git a/sdks/go/pkg/beam/runners/dataflow/translate.go
b/sdks/go/pkg/beam/runners/dataflow/translate.go
index c2d4f88a11c..440dcd7a9ab 100644
--- a/sdks/go/pkg/beam/runners/dataflow/translate.go
+++ b/sdks/go/pkg/beam/runners/dataflow/translate.go
@@ -24,6 +24,7 @@ import (
"github.com/apache/beam/sdks/go/pkg/beam"
"github.com/apache/beam/sdks/go/pkg/beam/core/graph"
+ "github.com/apache/beam/sdks/go/pkg/beam/core/graph/coder"
"github.com/apache/beam/sdks/go/pkg/beam/core/graph/window"
"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/exec"
"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/graphx"
@@ -32,7 +33,6 @@ import (
rnapi_pb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
"github.com/golang/protobuf/proto"
df "google.golang.org/api/dataflow/v1b3"
- "github.com/apache/beam/sdks/go/pkg/beam/core/graph/coder"
)
const (
diff --git a/sdks/go/pkg/beam/util/gcsx/gcs.go
b/sdks/go/pkg/beam/util/gcsx/gcs.go
index b39bd21514a..666a837fb0f 100644
--- a/sdks/go/pkg/beam/util/gcsx/gcs.go
+++ b/sdks/go/pkg/beam/util/gcsx/gcs.go
@@ -27,7 +27,9 @@ import (
"golang.org/x/oauth2/google"
"google.golang.org/api/googleapi"
+ "google.golang.org/api/option"
"google.golang.org/api/storage/v1"
+ ghttp "google.golang.org/api/transport/http"
)
// NewClient creates a new GCS client with default application credentials.
@@ -39,6 +41,15 @@ func NewClient(ctx context.Context, scope string)
(*storage.Service, error) {
return storage.New(cl)
}
+// NewUnauthenticatedClient creates a new GCS client without authentication.
+func NewUnauthenticatedClient(ctx context.Context) (*storage.Service, error) {
+ cl, _, err := ghttp.NewClient(ctx, option.WithoutAuthentication())
+ if err != nil {
+ return nil, fmt.Errorf("dialing: %v", err)
+ }
+ return storage.New(cl)
+}
+
// Upload writes the given content to GCS. If the specified bucket does not
// exist, it is created first. Returns the full path of the object.
func Upload(client *storage.Service, project, bucket, object string, r
io.Reader) (string, error) {
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 82537)
Time Spent: 1h (was: 50m)
> Go SDK GCS I/O shouldn't require credentials to read from public buckets
> ------------------------------------------------------------------------
>
> Key: BEAM-3893
> URL: https://issues.apache.org/jira/browse/BEAM-3893
> Project: Beam
> Issue Type: Bug
> Components: sdk-go
> Reporter: Henning Rohde
> Assignee: Henning Rohde
> Priority: Minor
> Time Spent: 1h
> Remaining Estimate: 0h
>
> We should handle the case where application default credentials are not
> available for public reads. Wordcount running a container fails currently:
> 2018/03/20 20:03:00 Failed to execute job: panic: panic: failed to create GCE
> client: google: could not find default credentials. See
> https://developers.google.com/accounts/docs/application-default-credentials
> for more information. goroutine 1 [running]:
> runtime/debug.Stack(0xc4201728a0, 0xc39460, 0xc4201c0120)
> /usr/local/go/src/runtime/debug/stack.go:24 +0xa7
> github.com/apache/beam/sdks/go/pkg/beam/core/util/reflectx.CallNoPanic.func1(0xc420172f80)
>
> /foo/src/github.com/apache/beam/sdks/go/pkg/beam/core/util/reflectx/call.go:98
> +0x6e
> panic(0xc39460, 0xc4201c0120)
> /usr/local/go/src/runtime/panic.go:491 +0x283
> github.com/apache/beam/sdks/go/pkg/beam/io/textio/gcs.New(0x12f3500,
> 0xc420014060, 0xc420094c40, 0x2)
>
> /foo/src/github.com/apache/beam/sdks/go/pkg/beam/io/textio/gcs/gcs.go:44
> +0x16e
> github.com/apache/beam/sdks/go/pkg/beam/io/textio.newFileSystem(0x12f3500,
> 0xc420014060, 0xc420094c40, 0x31, 0xc420172b50, 0x4bf4ef, 0xc37e60, 0xc37e60)
> /foo/src/github.com/apache/beam/sdks/go/pkg/beam/io/textio/textio.go:68
> +0xac
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
