lostluck commented on code in PR #40686: URL: https://github.com/apache/beam/pull/40686#discussion_r4331516463
########## sdks/go/pkg/beam/options/gcpopts/universe_test.go: ########## @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcpopts + +import ( + "context" + "errors" + "testing" + + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime" + "golang.org/x/oauth2/google" +) + +func TestGetUniverseDomain(t *testing.T) { + tests := []struct { + name string + env string + credsUD string + credsErr error + want string + wantCalls int // credential lookups across two calls + }{ + { + name: "env var takes precedence", + env: "env.example.com", + credsUD: "creds.example.com", + want: "env.example.com", + }, + { + name: "falls back to credentials and caches", + credsUD: "creds.example.com", + want: "creds.example.com", + wantCalls: 1, + }, + { + name: "credential errors are not cached", + credsErr: errors.New("no credentials"), + want: "", + wantCalls: 2, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + resetUniverseDomainCache(t) + t.Setenv(universeDomainEnvVar, test.env) + calls := 0 + orig := findDefaultCredentials + findDefaultCredentials = func(context.Context, ...string) (*google.Credentials, error) { + calls++ + if test.credsErr != nil { + return nil, test.credsErr + } + return &google.Credentials{ + UniverseDomainProvider: func() (string, error) { return test.credsUD, nil }, + }, nil + } + t.Cleanup(func() { findDefaultCredentials = orig }) + + for i := 0; i < 2; i++ { + if got := GetUniverseDomain(context.Background()); got != test.want { Review Comment: nit, use t.Context() here instead. ########## sdks/go/pkg/beam/runners/dataflow/dataflowlib/client_test.go: ########## @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dataflowlib + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/json" + "encoding/pem" + "os" + "path/filepath" + "testing" +) + +func TestNewClient_BasePath(t *testing.T) { + // Fake service account credentials in a non-default universe. The auth + // library parses the key when the client is created (non-default universes + // use self-signed JWTs), so it must be a valid key. No token is fetched + // until a request is made. + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("Failed generating key: %v", err) + } + der, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatalf("Failed marshaling key: %v", err) + } + content, err := json.Marshal(map[string]string{ + "type": "service_account", + "project_id": "test-project", + "client_email": "[email protected]", + "private_key_id": "test-key-id", + "private_key": string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der})), + "universe_domain": "creds.example.com", + }) + if err != nil { + t.Fatalf("Failed marshaling credentials: %v", err) + } + credFile := filepath.Join(t.TempDir(), "creds.json") + if err := os.WriteFile(credFile, content, 0600); err != nil { + t.Fatalf("Failed writing fake credential file: %v", err) + } + t.Setenv("GOOGLE_APPLICATION_CREDENTIALS", credFile) + + tests := []struct { + name string + endpoint string + env string + want string + }{ + { + // The client library doesn't read the credentials' universe domain, + // so this verifies that NewClient passes it through. + name: "universe domain from credentials", + want: "https://dataflow.creds.example.com/";, + }, + { + name: "endpoint takes precedence over universe domain", + endpoint: "http://localhost:8080/";, + env: "creds.example.com", + want: "http://localhost:8080/";, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Setenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN", test.env) + + client, err := NewClient(context.Background(), test.endpoint) Review Comment: Same nit, prefer t.Context() in tests. ########## sdks/go/pkg/beam/options/gcpopts/options.go: ########## @@ -34,8 +36,73 @@ var ( // Region is the GCP region where the job should be run. Region = flag.String("region", "", "GCP Region (required)") + + // GcsEndpoint overrides the Google Cloud Storage API endpoint. + GcsEndpoint = flag.String("gcs_endpoint", "", "The URL for the GCS API, "+ + "e.g. https://storage.googleapis.com/storage/v1/ (optional). If not set, the "+ + "endpoint is derived from the universe domain, which is read from "+ + "GOOGLE_CLOUD_UNIVERSE_DOMAIN or the default credentials.") ) +// GetGcsEndpoint returns the GCS endpoint override from the --gcs_endpoint +// flag. If the flag is unset, it falls back to the "gcs_endpoint" pipeline +// option, which is how the value reaches remote workers, where flags are not +// parsed. Returns an empty string if no override is configured. +func GetGcsEndpoint() string { + if *GcsEndpoint != "" { + return *GcsEndpoint + } + return runtime.GlobalOptions.Get("gcs_endpoint") +} + +const universeDomainEnvVar = "GOOGLE_CLOUD_UNIVERSE_DOMAIN" + +// findDefaultCredentials is a variable to allow overriding in tests. +var findDefaultCredentials = google.FindDefaultCredentials + +// cachedUniverseDomain caches the universe domain resolved from the default +// credentials, since resolving it may require a metadata server round trip +// and clients are created frequently. +var cachedUniverseDomain struct { + mu sync.Mutex + resolved bool + value string +} Review Comment: Beautiful use of an anonymous struct type. ########## sdks/go/pkg/beam/options/gcpopts/universe_test.go: ########## @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcpopts + +import ( + "context" + "errors" + "testing" + + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime" + "golang.org/x/oauth2/google" +) + +func TestGetUniverseDomain(t *testing.T) { + tests := []struct { + name string + env string + credsUD string + credsErr error + want string + wantCalls int // credential lookups across two calls + }{ + { + name: "env var takes precedence", + env: "env.example.com", + credsUD: "creds.example.com", + want: "env.example.com", + }, + { + name: "falls back to credentials and caches", + credsUD: "creds.example.com", + want: "creds.example.com", + wantCalls: 1, + }, + { + name: "credential errors are not cached", + credsErr: errors.New("no credentials"), + want: "", + wantCalls: 2, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + resetUniverseDomainCache(t) + t.Setenv(universeDomainEnvVar, test.env) + calls := 0 + orig := findDefaultCredentials + findDefaultCredentials = func(context.Context, ...string) (*google.Credentials, error) { + calls++ + if test.credsErr != nil { + return nil, test.credsErr + } + return &google.Credentials{ + UniverseDomainProvider: func() (string, error) { return test.credsUD, nil }, + }, nil + } + t.Cleanup(func() { findDefaultCredentials = orig }) + + for i := 0; i < 2; i++ { Review Comment: Add a comment why we're doing repetitions here please. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
