This is an automated email from the ASF dual-hosted git repository.
pjfanning pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/pekko-connectors.git
The following commit(s) were added to refs/heads/main by this push:
new caa980a01 Google: add top-level project-id setting with environment
fallback (#1926)
caa980a01 is described below
commit caa980a0123bc24df081e468eacbfa11ccb1e4ee
Author: PJ Fanning <[email protected]>
AuthorDate: Sun Sep 6 20:57:42 2026 +0100
Google: add top-level project-id setting with environment fallback (#1926)
The project id was always taken from the credentials provider, so the only
way
to target a project other than the one owning the credentials' principal
was to
override GoogleSettings.projectId programmatically. That is a common setup:
a
service account in one project is frequently granted access to resources in
another.
Add two top-level settings under pekko.connectors.google:
* project-id, which takes precedence over the credentials provider;
* default-project-id, used when neither of the above supplies a value, and
populated from the GOOGLE_CLOUD_PROJECT, GCLOUD_PROJECT and GCP_PROJECT
environment variables that GCP runtimes and GKE Workload Identity set.
Both are additive and default to empty, so existing configurations resolve
the
same project id as before. Configs built without the new keys keep working.
Co-authored-by: Claude Opus 5 (1M context) <[email protected]>
---
docs/src/main/paradox/google-common.md | 18 +++++
google-common/src/main/resources/reference.conf | 17 +++++
.../stream/connectors/google/GoogleSettings.scala | 16 ++++-
.../connectors/google/GoogleSettingsSpec.scala | 81 ++++++++++++++++++++++
4 files changed, 131 insertions(+), 1 deletion(-)
diff --git a/docs/src/main/paradox/google-common.md
b/docs/src/main/paradox/google-common.md
index c22fbbc50..cde562891 100644
--- a/docs/src/main/paradox/google-common.md
+++ b/docs/src/main/paradox/google-common.md
@@ -36,6 +36,24 @@ Credentials will be loaded automatically:
Credentials can also be specified manually in your configuration file.
+## Project id
+
+The project id used for requests is resolved in this order:
+
+1. The `pekko.connectors.google.project-id` setting, if it is non-empty;
+2. the project id supplied by the configured credentials provider, if it is
non-empty;
+3. the `pekko.connectors.google.default-project-id` setting, which reads the
`GOOGLE_CLOUD_PROJECT`,
+ `GCLOUD_PROJECT` and `GCP_PROJECT` environment variables by default.
+
+Setting `project-id` explicitly is useful because the project that owns the
principal your credentials
+authenticate as need not be the project whose resources you want to access; a
service account in one
+project is often granted access to another. The environment variables behind
`default-project-id` are
+commonly set for you by GCP runtimes such as App Engine, Cloud Run, Cloud
Functions and by GKE
+Workload Identity.
+
+The resolved value is available as `GoogleSettings.projectId` and can still be
overridden per stream
+with `withProjectId`.
+
## Accessing settings
@apidoc[GoogleSettings$] provides methods to retrieve settings from your
configuration and @apidoc[GoogleAttributes$] to access the settings attached to
a stream.
diff --git a/google-common/src/main/resources/reference.conf
b/google-common/src/main/resources/reference.conf
index d18d78579..f75a84529 100644
--- a/google-common/src/main/resources/reference.conf
+++ b/google-common/src/main/resources/reference.conf
@@ -66,6 +66,23 @@ pekko.connectors.google {
}
}
+ # The GCP project id used for requests to Google APIs.
+ # If empty, the project id reported by the configured credentials provider
is used, and if that is
+ # empty too, `default-project-id` is used.
+ # Set this when the project you want to access is not the project that owns
the principal the
+ # credentials authenticate as, which is common when a service account in one
project has been
+ # granted access to resources in another.
+ project-id = ""
+
+ # Fallback project id, used when neither `project-id` nor the credentials
provider supplies one.
+ # These environment variables are commonly set for you by GCP runtimes such
as App Engine,
+ # Cloud Run, Cloud Functions and by GKE Workload Identity.
+ # Later entries take precedence over earlier ones.
+ default-project-id = ""
+ default-project-id = ${?GCP_PROJECT}
+ default-project-id = ${?GCLOUD_PROJECT}
+ default-project-id = ${?GOOGLE_CLOUD_PROJECT}
+
# Standard query parameters for all Google APIs sent with every request
user-ip = ""
quota-user = ""
diff --git
a/google-common/src/main/scala/org/apache/pekko/stream/connectors/google/GoogleSettings.scala
b/google-common/src/main/scala/org/apache/pekko/stream/connectors/google/GoogleSettings.scala
index 3a4989151..484461707 100644
---
a/google-common/src/main/scala/org/apache/pekko/stream/connectors/google/GoogleSettings.scala
+++
b/google-common/src/main/scala/org/apache/pekko/stream/connectors/google/GoogleSettings.scala
@@ -44,7 +44,21 @@ object GoogleSettings {
val credentials = Credentials(c.getConfig("credentials"))
val requestSettings = RequestSettings(c)
- GoogleSettings(credentials.projectId, credentials, requestSettings)
+ GoogleSettings(resolveProjectId(c, credentials), credentials,
requestSettings)
+ }
+
+ /**
+ * The `project-id` setting wins if it is set, otherwise the project id
supplied by the credentials
+ * provider, otherwise the `default-project-id` setting, which is read from
the environment by default.
+ */
+ private def resolveProjectId(c: Config, credentials: Credentials): String = {
+ def setting(path: String) =
+ if (c.hasPath(path)) Some(c.getString(path)).filter(_.nonEmpty) else None
+
+ setting("project-id")
+ .orElse(Some(credentials.projectId).filter(_.nonEmpty))
+ .orElse(setting("default-project-id"))
+ .getOrElse("")
}
/**
diff --git
a/google-common/src/test/scala/org/apache/pekko/stream/connectors/google/GoogleSettingsSpec.scala
b/google-common/src/test/scala/org/apache/pekko/stream/connectors/google/GoogleSettingsSpec.scala
index 0f9e6eeee..c68ecebf0 100644
---
a/google-common/src/test/scala/org/apache/pekko/stream/connectors/google/GoogleSettingsSpec.scala
+++
b/google-common/src/test/scala/org/apache/pekko/stream/connectors/google/GoogleSettingsSpec.scala
@@ -51,6 +51,87 @@ class GoogleSettingsSpec
""".stripMargin)
.resolve)
+ private def mkSettings(more: String): GoogleSettings =
+ GoogleSettings(
+ ConfigFactory
+ .parseString(more)
+
.withFallback(ConfigFactory.defaultReference().getConfig(GoogleSettings.ConfigPath))
+ .resolve)
+
+ it should "prefer the project-id setting over the one from the credentials
provider" in {
+ mkSettings("""
+ |project-id = "explicit-project"
+ |default-project-id = "env-project"
+ |credentials {
+ | provider = access-token
+ | access-token {
+ | project-id = "credentials-project"
+ | token = "yyyy.c.an-access-token"
+ | }
+ |}
+ """.stripMargin).projectId shouldEqual "explicit-project"
+ }
+
+ it should "fall back to the project id from the credentials provider" in {
+ mkSettings("""
+ |project-id = ""
+ |default-project-id = "env-project"
+ |credentials {
+ | provider = access-token
+ | access-token {
+ | project-id = "credentials-project"
+ | token = "yyyy.c.an-access-token"
+ | }
+ |}
+ """.stripMargin).projectId shouldEqual "credentials-project"
+ }
+
+ it should "fall back to default-project-id when no other project id is
available" in {
+ mkSettings("""
+ |project-id = ""
+ |default-project-id = "env-project"
+ |credentials {
+ | provider = access-token
+ | access-token {
+ | project-id = ""
+ | token = "yyyy.c.an-access-token"
+ | }
+ |}
+ """.stripMargin).projectId shouldEqual "env-project"
+ }
+
+ it should "leave the project id empty when nothing supplies one" in {
+ mkSettings("""
+ |project-id = ""
+ |default-project-id = ""
+ |credentials {
+ | provider = access-token
+ | access-token {
+ | project-id = ""
+ | token = "yyyy.c.an-access-token"
+ | }
+ |}
+ """.stripMargin).projectId shouldEqual ""
+ }
+
+ it should "resolve the project id from a config that predates the project-id
settings" in {
+ val legacy = ConfigFactory
+ .parseString("""
+ |credentials {
+ | provider = access-token
+ | access-token {
+ | project-id = "credentials-project"
+ | token = "yyyy.c.an-access-token"
+ | }
+ |}
+ """.stripMargin)
+
.withFallback(ConfigFactory.defaultReference().getConfig(GoogleSettings.ConfigPath))
+ .resolve
+ .withoutPath("project-id")
+ .withoutPath("default-project-id")
+ GoogleSettings(legacy).projectId shouldEqual "credentials-project"
+ }
+
it should "skip parsing forward-proxy when optional environment overrides
exist but aren't set" in {
@nowarn("msg=possible missing interpolator: detected an interpolated
expression")
val config = """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
