This is an automated email from the ASF dual-hosted git repository.
damondouglas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 00a55272eaa [RRIO] [Testing] Mock API integration tests (#29236)
00a55272eaa is described below
commit 00a55272eaac451c3f364c4071e4af1ab56b88bd
Author: Damon <[email protected]>
AuthorDate: Wed Nov 1 10:09:14 2023 -0700
[RRIO] [Testing] Mock API integration tests (#29236)
* Update README with integration test instructions
* Replace refresher with human readable ids
* Impl integration tests; refactor code errors
* Add missing code comment
* Fix whitespace
---
.test-infra/mock-apis/README.md | 56 ++++--
.test-infra/mock-apis/go.mod | 6 +-
.test-infra/mock-apis/go.sum | 12 +-
.../configmap.yaml | 24 ---
.../configmap.yaml | 8 +-
.../deployment.yaml | 6 +-
.../kustomization.yaml | 4 +-
.../configmap.yaml | 7 +-
.../deployment.yaml | 6 +-
.../kustomization.yaml | 4 +-
.../deployment.yaml | 27 ---
.../kustomization.yaml | 34 ----
.../src/main/go/internal/service/echo/echo.go | 47 +++--
.../src/main/go/test/integration/echo/echo_test.go | 220 +++++++++++++++++++++
.../src/main/go/test/integration/integration.go | 32 +++
.../mock-apis/src/main/go/test/integration/vars.go | 49 +++++
16 files changed, 402 insertions(+), 140 deletions(-)
diff --git a/.test-infra/mock-apis/README.md b/.test-infra/mock-apis/README.md
index df34757b770..9c4911a0d63 100644
--- a/.test-infra/mock-apis/README.md
+++ b/.test-infra/mock-apis/README.md
@@ -75,7 +75,45 @@ go test ./src/main/go/internal/...
## Integration
-TODO: See https://github.com/apache/beam/issues/28859
+Integration tests require the following values.
+
+### Quota ID
+
+Each allocated quota corresponds to a unique ID known as the Quota ID.
+There exists a one-to-one relationship between the allocated quota and
+the
+[infrastructure/kubernetes/refresher/overlays](infrastructure/kubernetes/refresher/overlays).
+
+To query the Kubernetes cluster for allocated Quota IDs:
+```
+kubectl get deploy --selector=app.kubernetes.io/name=refresher -o
custom-columns='QUOTA_ID:.metadata.labels.quota-id'
+```
+
+### Service Endpoint
+
+To list available endpoints, run:
+
+```
+kubectl get svc
-o=custom-columns='NAME:.metadata.name,HOST:.status.loadBalancer.ingress[*].ip,PORT_NAME:.spec.ports[*].name,PORT:.spec.ports[*].port'
+```
+
+You should see something similar to:
+
+```
+NAME HOST PORT_NAME PORT
+echo 10.n.n.n grpc,http 50051,8080
+```
+
+When running tests locally, you will need to first run:
+```
+kubectl port-forward service/echo 50051:50051 8080:8080
+```
+
+which allows you to access the gRPC via `localhost:50051` and the HTTP via
+`http://localhost:8080/v1/echo`.
+
+When running tests on Dataflow, you supply `10.n.n.n:50051` for gRPC and
+`http://10.n.n.n:8080/v1/echo` for HTTP.
# Local Usage
@@ -186,24 +224,14 @@ The Refresher service relies on
[kustomize](https://kustomize.io) overlays
which are located at
[infrastructure/kubernetes/refresher/overlays](infrastructure/kubernetes/refresher/overlays).
Each folder contained in
[infrastructure/kubernetes/refresher/overlays](infrastructure/kubernetes/refresher/overlays)
-corresponds to an individual Refresher instance that is identified by the UUID.
-You will need to deploy each one individually.
+corresponds to an individual Refresher instance that is identified by a unique
+string id. You will need to deploy each one individually.
For example:
```
-kubectl kustomize
infrastructure/kubernetes/refresher/overlays/f588787b-28f8-4e5f-8335-f862379daf59
| ko resolve -f - | kubectl apply -f -
+kubectl kustomize
infrastructure/kubernetes/refresher/overlays/echo-should-never-exceed-quota |
ko resolve -f - | kubectl apply -f -
```
Like previously, you may see "Does not have minimum availability" message
showing on the status. It may take some time for GKE autopilot
to scale the node pool.
-
-## Additional note for creating a new Refresher service instance
-
-Each Refresher service instance relies on a unique UUID, where
-the [kustomize](https://kustomize.io) overlay replaces in the
-[infrastructure/kubernetes/refresher/base](infrastructure/kubernetes/refresher/base)
-template.
-
-You can copy the entire folder and paste into a new one with a unique UUID
-and then perform a find-replace of the old UUID with the new one.
diff --git a/.test-infra/mock-apis/go.mod b/.test-infra/mock-apis/go.mod
index ef2953f27c5..cc65cfbaac7 100644
--- a/.test-infra/mock-apis/go.mod
+++ b/.test-infra/mock-apis/go.mod
@@ -45,11 +45,11 @@ require (
github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
go.opencensus.io v0.24.0 // indirect
- golang.org/x/crypto v0.13.0 // indirect
- golang.org/x/net v0.15.0 // indirect
+ golang.org/x/crypto v0.14.0 // indirect
+ golang.org/x/net v0.17.0 // indirect
golang.org/x/oauth2 v0.12.0 // indirect
golang.org/x/sync v0.3.0 // indirect
- golang.org/x/sys v0.12.0 // indirect
+ golang.org/x/sys v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
google.golang.org/api v0.128.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
diff --git a/.test-infra/mock-apis/go.sum b/.test-infra/mock-apis/go.sum
index 74b587a7244..a928e3dae2f 100644
--- a/.test-infra/mock-apis/go.sum
+++ b/.test-infra/mock-apis/go.sum
@@ -101,8 +101,8 @@ golang.org/x/crypto
v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod
h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod
h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod
h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
-golang.org/x/crypto v0.13.0/go.mod
h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod
h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod
h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod
h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod
h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -121,8 +121,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod
h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod
h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod
h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod
h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod
h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod
h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4=
@@ -144,8 +144,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod
h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod
h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod
h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod
h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod
h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod
h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
diff --git
a/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/123079b5-1e58-4b28-a185-66702e2b10c3/configmap.yaml
b/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/123079b5-1e58-4b28-a185-66702e2b10c3/configmap.yaml
deleted file mode 100644
index bf87b0646ea..00000000000
---
a/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/123079b5-1e58-4b28-a185-66702e2b10c3/configmap.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Configures patch for ../base/configmap.yaml
-# See
https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/patches/
-
-- op: replace
- path: /metadata/labels/quota-id
- value: 123079b5-1e58-4b28-a185-66702e2b10c3
-- op: replace
- path: /data/QUOTA_ID
- value: 123079b5-1e58-4b28-a185-66702e2b10c3
diff --git
a/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/e1064224-3671-46fe-971d-47887fac3d4c/configmap.yaml
b/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/echo-should-exceed-quota/configmap.yaml
similarity index 87%
rename from
.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/e1064224-3671-46fe-971d-47887fac3d4c/configmap.yaml
rename to
.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/echo-should-exceed-quota/configmap.yaml
index 71b19b257f2..274ae43ebb8 100644
---
a/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/e1064224-3671-46fe-971d-47887fac3d4c/configmap.yaml
+++
b/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/echo-should-exceed-quota/configmap.yaml
@@ -18,7 +18,11 @@
- op: replace
path: /metadata/labels/quota-id
- value: e1064224-3671-46fe-971d-47887fac3d4c
+ value: echo-should-exceed-quota
- op: replace
path: /data/QUOTA_ID
- value: e1064224-3671-46fe-971d-47887fac3d4c
+ value: echo-should-exceed-quota
+- op: replace
+ path: /data/QUOTA_SIZE
+ # We need at least 1
+ value: "1"
diff --git
a/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/123079b5-1e58-4b28-a185-66702e2b10c3/deployment.yaml
b/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/echo-should-exceed-quota/deployment.yaml
similarity index 88%
rename from
.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/123079b5-1e58-4b28-a185-66702e2b10c3/deployment.yaml
rename to
.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/echo-should-exceed-quota/deployment.yaml
index 7ad531ce7d8..e903a6c7c29 100644
---
a/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/123079b5-1e58-4b28-a185-66702e2b10c3/deployment.yaml
+++
b/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/echo-should-exceed-quota/deployment.yaml
@@ -18,10 +18,10 @@
- op: replace
path: /metadata/labels/quota-id
- value: 123079b5-1e58-4b28-a185-66702e2b10c3
+ value: echo-should-exceed-quota
- op: replace
path: /spec/selector/matchLabels/quota-id
- value: 123079b5-1e58-4b28-a185-66702e2b10c3
+ value: echo-should-exceed-quota
- op: replace
path: /spec/template/metadata/labels/quota-id
- value: 123079b5-1e58-4b28-a185-66702e2b10c3
+ value: echo-should-exceed-quota
diff --git
a/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/e1064224-3671-46fe-971d-47887fac3d4c/kustomization.yaml
b/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/echo-should-exceed-quota/kustomization.yaml
similarity index 92%
rename from
.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/e1064224-3671-46fe-971d-47887fac3d4c/kustomization.yaml
rename to
.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/echo-should-exceed-quota/kustomization.yaml
index 3dd6ff160ab..9330ea4c6c7 100644
---
a/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/e1064224-3671-46fe-971d-47887fac3d4c/kustomization.yaml
+++
b/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/echo-should-exceed-quota/kustomization.yaml
@@ -15,12 +15,12 @@
# Configures the overlay for
.test-infra/mock-apis/infrastructure/kubernetes/refresher/base
# Using the Quota Id:
-# e1064224-3671-46fe-971d-47887fac3d4c
+# echo-should-exceed-quota
resources:
- ../../base
-nameSuffix: -e1064224-3671-46fe-971d-47887fac3d4c
+nameSuffix: -echo-should-exceed-quota
patches:
- path: configmap.yaml
diff --git
a/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/f588787b-28f8-4e5f-8335-f862379daf59/configmap.yaml
b/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/echo-should-never-exceed-quota/configmap.yaml
similarity index 87%
rename from
.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/f588787b-28f8-4e5f-8335-f862379daf59/configmap.yaml
rename to
.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/echo-should-never-exceed-quota/configmap.yaml
index b6a12f7f133..409d83a8126 100644
---
a/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/f588787b-28f8-4e5f-8335-f862379daf59/configmap.yaml
+++
b/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/echo-should-never-exceed-quota/configmap.yaml
@@ -18,7 +18,10 @@
- op: replace
path: /metadata/labels/quota-id
- value: f588787b-28f8-4e5f-8335-f862379daf59
+ value: echo-should-never-exceed-quota
- op: replace
path: /data/QUOTA_ID
- value: f588787b-28f8-4e5f-8335-f862379daf59
+ value: echo-should-never-exceed-quota
+- op: replace
+ path: /data/QUOTA_SIZE
+ value: "1000000000"
diff --git
a/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/e1064224-3671-46fe-971d-47887fac3d4c/deployment.yaml
b/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/echo-should-never-exceed-quota/deployment.yaml
similarity index 88%
rename from
.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/e1064224-3671-46fe-971d-47887fac3d4c/deployment.yaml
rename to
.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/echo-should-never-exceed-quota/deployment.yaml
index 9f13ec4b784..d550adf0204 100644
---
a/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/e1064224-3671-46fe-971d-47887fac3d4c/deployment.yaml
+++
b/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/echo-should-never-exceed-quota/deployment.yaml
@@ -18,10 +18,10 @@
- op: replace
path: /metadata/labels/quota-id
- value: e1064224-3671-46fe-971d-47887fac3d4c
+ value: echo-should-never-exceed-quota
- op: replace
path: /spec/selector/matchLabels/quota-id
- value: e1064224-3671-46fe-971d-47887fac3d4c
+ value: echo-should-never-exceed-quota
- op: replace
path: /spec/template/metadata/labels/quota-id
- value: e1064224-3671-46fe-971d-47887fac3d4c
+ value: echo-should-never-exceed-quota
diff --git
a/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/123079b5-1e58-4b28-a185-66702e2b10c3/kustomization.yaml
b/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/echo-should-never-exceed-quota/kustomization.yaml
similarity index 92%
rename from
.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/123079b5-1e58-4b28-a185-66702e2b10c3/kustomization.yaml
rename to
.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/echo-should-never-exceed-quota/kustomization.yaml
index 496e5354463..1f8d23ba01b 100644
---
a/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/123079b5-1e58-4b28-a185-66702e2b10c3/kustomization.yaml
+++
b/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/echo-should-never-exceed-quota/kustomization.yaml
@@ -15,12 +15,12 @@
# Configures the overlay for
.test-infra/mock-apis/infrastructure/kubernetes/refresher/base
# Using the Quota Id:
-# 123079b5-1e58-4b28-a185-66702e2b10c3
+# echo-should-never-exceed-quota
resources:
- ../../base
-nameSuffix: -123079b5-1e58-4b28-a185-66702e2b10c3
+nameSuffix: -echo-should-never-exceed-quota
patches:
- path: configmap.yaml
diff --git
a/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/f588787b-28f8-4e5f-8335-f862379daf59/deployment.yaml
b/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/f588787b-28f8-4e5f-8335-f862379daf59/deployment.yaml
deleted file mode 100644
index 214ed9634a8..00000000000
---
a/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/f588787b-28f8-4e5f-8335-f862379daf59/deployment.yaml
+++ /dev/null
@@ -1,27 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Configures patch for ../base/deployment.yaml
-# See
https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/patches/
-
-- op: replace
- path: /metadata/labels/quota-id
- value: f588787b-28f8-4e5f-8335-f862379daf59
-- op: replace
- path: /spec/selector/matchLabels/quota-id
- value: f588787b-28f8-4e5f-8335-f862379daf59
-- op: replace
- path: /spec/template/metadata/labels/quota-id
- value: f588787b-28f8-4e5f-8335-f862379daf59
diff --git
a/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/f588787b-28f8-4e5f-8335-f862379daf59/kustomization.yaml
b/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/f588787b-28f8-4e5f-8335-f862379daf59/kustomization.yaml
deleted file mode 100644
index 5198e423819..00000000000
---
a/.test-infra/mock-apis/infrastructure/kubernetes/refresher/overlays/f588787b-28f8-4e5f-8335-f862379daf59/kustomization.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Configures the overlay for
.test-infra/mock-apis/infrastructure/kubernetes/refresher/base
-# Using the Quota Id:
-# f588787b-28f8-4e5f-8335-f862379daf59
-
-resources:
-- ../../base
-
-nameSuffix: -f588787b-28f8-4e5f-8335-f862379daf59
-
-patches:
-- path: configmap.yaml
- target:
- kind: ConfigMap
- name: refresher
-
-- path: deployment.yaml
- target:
- kind: Deployment
- name: refresher
diff --git a/.test-infra/mock-apis/src/main/go/internal/service/echo/echo.go
b/.test-infra/mock-apis/src/main/go/internal/service/echo/echo.go
index d958d18576e..d0682551775 100644
--- a/.test-infra/mock-apis/src/main/go/internal/service/echo/echo.go
+++ b/.test-infra/mock-apis/src/main/go/internal/service/echo/echo.go
@@ -39,7 +39,7 @@ import (
const (
metricsNamePrefix = "echo"
echoPath = "/proto.echo.v1.EchoService/Echo"
- echoPathAlias = "/v1/echo"
+ PathAlias = "/v1/echo"
healthPath = "/grpc.health.v1.Health/Check"
healthPathAlias = "/v1/healthz"
)
@@ -58,6 +58,11 @@ func Register(s *grpc.Server, opts *Options) (http.Handler,
error) {
Name: reflect.TypeOf((*echo)(nil)).PkgPath(),
})
}
+ var attrs []any
+ for _, attr := range opts.LoggingAttrs {
+ attrs = append(attrs, attr)
+ }
+ opts.Logger = opts.Logger.With(attrs...)
srv := &echo{
opts: opts,
}
@@ -77,7 +82,7 @@ type echo struct {
// ServeHTTP implements http.Handler, allowing echo to support HTTP clients in
addition to gRPC.
func (srv *echo) ServeHTTP(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
- case echoPath, echoPathAlias:
+ case echoPath, PathAlias:
srv.httpHandler(w, r)
case healthPath, healthPathAlias:
srv.checkHandler(w, r)
@@ -104,7 +109,7 @@ func (srv *echo) checkHandler(w http.ResponseWriter, r
*http.Request) {
return
}
if err := json.NewEncoder(w).Encode(resp); err != nil {
- srv.opts.Logger.LogAttrs(context.Background(), slog.LevelError,
err.Error(), srv.opts.LoggingAttrs...)
+ srv.opts.Logger.Log(r.Context(), slog.LevelError, err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
@@ -113,7 +118,7 @@ func (srv *echo) checkHandler(w http.ResponseWriter, r
*http.Request) {
func (srv *echo) Watch(request *grpc_health_v1.HealthCheckRequest, server
grpc_health_v1.Health_WatchServer) error {
resp, err := srv.Check(server.Context(), request)
if err != nil {
- srv.opts.Logger.LogAttrs(context.Background(), slog.LevelError,
err.Error(), srv.opts.LoggingAttrs...)
+ srv.opts.Logger.Log(server.Context(), slog.LevelError,
err.Error())
return err
}
return server.Send(resp)
@@ -128,7 +133,7 @@ func (srv *echo) Echo(ctx context.Context, request
*echov1.EchoRequest) (*echov1
return nil, status.Errorf(codes.NotFound, "error: source not
found: %s, err %v", request.Id, err)
}
if err != nil {
- srv.opts.Logger.LogAttrs(context.Background(), slog.LevelError,
err.Error(), srv.opts.LoggingAttrs...)
+ srv.opts.Logger.Log(ctx, slog.LevelError, err.Error())
return nil, status.Errorf(codes.Internal, "error: encountered
from cache for resource: %srv, err %v", request.Id, err)
}
@@ -137,7 +142,7 @@ func (srv *echo) Echo(ctx context.Context, request
*echov1.EchoRequest) (*echov1
}
if v < 0 {
- return nil, status.Errorf(codes.ResourceExhausted, "error:
resource exhausted for: %srv", request.Id)
+ return nil, status.Errorf(codes.ResourceExhausted, "error:
resource exhausted for: %s", request.Id)
}
return &echov1.EchoResponse{
@@ -154,7 +159,7 @@ func (srv *echo) writeMetric(ctx context.Context, id
string, value int64) error
Timestamp: time.Now(),
Value: value + 1,
}); err != nil {
- srv.opts.Logger.LogAttrs(context.Background(), slog.LevelError,
err.Error(), srv.opts.LoggingAttrs...)
+ srv.opts.Logger.Log(ctx, slog.LevelError, err.Error())
}
return nil
}
@@ -163,23 +168,29 @@ func (srv *echo) httpHandler(w http.ResponseWriter, r
*http.Request) {
var body *echov1.EchoRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
err = fmt.Errorf("error decoding request body, payload field of
%T needs to be base64 encoded, error: %w", body, err)
- srv.opts.Logger.LogAttrs(context.Background(), slog.LevelError,
err.Error(), srv.opts.LoggingAttrs...)
+ srv.opts.Logger.Log(r.Context(), slog.LevelError, err.Error())
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
resp, err := srv.Echo(r.Context(), body)
- if status.Code(err) == http.StatusNotFound {
- http.Error(w, err.Error(), http.StatusNotFound)
- return
- }
- if err != nil {
- http.Error(w, err.Error(), http.StatusInternalServerError)
- return
- }
-
- if err := json.NewEncoder(w).Encode(resp); err != nil {
+ switch status.Code(err) {
+ case codes.OK:
+ if err := json.NewEncoder(w).Encode(resp); err != nil {
+ srv.opts.Logger.Log(r.Context(), slog.LevelError,
err.Error())
+ http.Error(w, err.Error(),
http.StatusInternalServerError)
+ }
+ case codes.InvalidArgument:
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ case codes.DeadlineExceeded:
+ http.Error(w, err.Error(), http.StatusRequestTimeout)
+ case codes.NotFound:
+ http.Error(w, err.Error(), http.StatusNotFound)
+ case codes.ResourceExhausted:
+ http.Error(w, err.Error(), http.StatusTooManyRequests)
+ default:
+ srv.opts.Logger.Log(r.Context(), slog.LevelError, err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
diff --git
a/.test-infra/mock-apis/src/main/go/test/integration/echo/echo_test.go
b/.test-infra/mock-apis/src/main/go/test/integration/echo/echo_test.go
new file mode 100644
index 00000000000..73ad597c7d3
--- /dev/null
+++ b/.test-infra/mock-apis/src/main/go/test/integration/echo/echo_test.go
@@ -0,0 +1,220 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Tests for the src/main/go/cmd/service/echo service.
+package echo
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net/http"
+ "regexp"
+ "testing"
+ "time"
+
+ echov1
"github.com/apache/beam/test-infra/mock-apis/src/main/go/internal/proto/echo/v1"
+
"github.com/apache/beam/test-infra/mock-apis/src/main/go/internal/service/echo"
+
"github.com/apache/beam/test-infra/mock-apis/src/main/go/test/integration"
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/credentials/insecure"
+ "google.golang.org/grpc/status"
+)
+
+const (
+ // QuotaIds below correspond to:
+ // kubectl get deploy --selector=app.kubernetes.io/tag=refresher -o
custom-columns='QUOTA_ID:.metadata.labels.quota-id'
+ // See
https://github.com/apache/beam/tree/master/.test-infra/mock-apis#writing-integration-tests
+ shouldExceedQuotaId = "echo-should-exceed-quota"
+ shouldNeverExceedQuotaId = "echo-should-never-exceed-quota"
+ shouldNotExistId = "should-not-exist"
+)
+
+var (
+ grpcOpts = []grpc.DialOption{
+ grpc.WithTransportCredentials(insecure.NewCredentials()),
+ grpc.WithBlock(),
+ }
+
+ timeout = time.Second * 3
+)
+
+func TestEcho(t *testing.T) {
+ payload := []byte("payload")
+
+ for _, tt := range []struct {
+ tag string
+ quotaId string
+ client echov1.EchoServiceClient
+ want *echov1.EchoResponse
+ wantErr error
+ }{
+ {
+ tag: "http",
+ quotaId: shouldExceedQuotaId,
+ client: withHttp(t),
+ wantErr: errors.New("429 Too Many Requests"),
+ },
+ {
+ tag: "grpc",
+ quotaId: shouldExceedQuotaId,
+ client: withGrpc(t),
+ wantErr: status.Error(codes.ResourceExhausted, "error:
resource exhausted for: echo-should-exceed-quota"),
+ },
+ {
+ tag: "http",
+ quotaId: shouldNotExistId,
+ client: withHttp(t),
+ wantErr: errors.New("404 Not Found"),
+ },
+ {
+ tag: "grpc",
+ quotaId: shouldNotExistId,
+ client: withGrpc(t),
+ wantErr: status.Error(codes.NotFound, "error: source
not found: should-not-exist, err resource does not exist"),
+ },
+ {
+ tag: "http",
+ quotaId: shouldNeverExceedQuotaId,
+ client: withHttp(t),
+ want: &echov1.EchoResponse{
+ Id: shouldNeverExceedQuotaId,
+ Payload: payload,
+ },
+ },
+ {
+ tag: "grpc",
+ quotaId: shouldNeverExceedQuotaId,
+ client: withGrpc(t),
+ want: &echov1.EchoResponse{
+ Id: shouldNeverExceedQuotaId,
+ Payload: payload,
+ },
+ },
+ } {
+ t.Run(fmt.Sprintf("%s/%s", tt.quotaId, tt.tag), func(t
*testing.T) {
+ ctx, cancel := withTimeout()
+ defer cancel()
+
+ req := &echov1.EchoRequest{
+ Id: tt.quotaId,
+ Payload: payload,
+ }
+
+ var resps []*echov1.EchoResponse
+ var errs []error
+
+ for i := 0; i < 3; i++ {
+ resp, err := tt.client.Echo(ctx, req)
+ if err != nil {
+ errs = append(errs, err)
+ }
+ if resp != nil {
+ resps = append(resps, resp)
+ }
+ }
+
+ if tt.wantErr != nil && len(errs) == 0 {
+ t.Errorf("Echo(%+v) err = nil, wantErr = %v",
req, tt.wantErr)
+ return
+ }
+
+ for _, err := range errs {
+ if diff := cmp.Diff(tt.wantErr.Error(),
err.Error()); diff != "" {
+ t.Errorf("Echo(%+v) err mismatch (-want
+got)\n%s", req, diff)
+ }
+ }
+
+ if tt.want != nil {
+ for _, resp := range resps {
+ if diff := cmp.Diff(tt.want, resp,
cmpopts.IgnoreUnexported(echov1.EchoResponse{})); diff != "" {
+ t.Errorf("Echo(%+v) mismatch
(-want +got)\n%s", req, diff)
+ }
+ }
+ }
+
+ })
+ }
+}
+
+func TestMain(m *testing.M) {
+ integration.Run(m)
+}
+
+func withGrpc(t *testing.T) echov1.EchoServiceClient {
+ t.Helper()
+ ctx, cancel := withTimeout()
+ defer cancel()
+
+ conn, err := grpc.DialContext(ctx, *integration.GRPCServiceEndpoint,
grpcOpts...)
+ if err != nil {
+ t.Fatalf("DialContext(%s) err %v",
*integration.GRPCServiceEndpoint, err)
+ }
+ t.Cleanup(func() {
+ if err := conn.Close(); err != nil {
+ t.Fatal(err)
+ }
+ })
+
+ return echov1.NewEchoServiceClient(conn)
+}
+
+type httpCaller struct {
+ rawUrl string
+}
+
+func (h *httpCaller) Echo(ctx context.Context, in *echov1.EchoRequest, _
...grpc.CallOption) (*echov1.EchoResponse, error) {
+ ctx, cancel := withTimeout()
+ defer cancel()
+ buf := bytes.Buffer{}
+ if err := json.NewEncoder(&buf).Encode(in); err != nil {
+ return nil, err
+ }
+
+ resp, err := http.Post(h.rawUrl, "application/json", &buf)
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.StatusCode > 299 {
+ return nil, errors.New(resp.Status)
+ }
+
+ var result *echov1.EchoResponse
+ if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+ return nil, err
+ }
+ return result, nil
+}
+
+func withHttp(t *testing.T) echov1.EchoServiceClient {
+ p := regexp.MustCompile(`^http://`)
+ rawUrl := fmt.Sprint(*integration.HTTPServiceEndpoint, echo.PathAlias)
+ if !p.MatchString(rawUrl) {
+ t.Fatalf("missing 'http(s)' scheme from %s",
*integration.HTTPServiceEndpoint)
+ }
+ return &httpCaller{
+ rawUrl: rawUrl,
+ }
+}
+
+func withTimeout() (context.Context, context.CancelFunc) {
+ return context.WithTimeout(context.Background(), timeout)
+}
diff --git a/.test-infra/mock-apis/src/main/go/test/integration/integration.go
b/.test-infra/mock-apis/src/main/go/test/integration/integration.go
new file mode 100644
index 00000000000..777225061ea
--- /dev/null
+++ b/.test-infra/mock-apis/src/main/go/test/integration/integration.go
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package integration provides functionality that needs to be shared between
all
+// integration tests.
+package integration
+
+import (
+ "flag"
+ "os"
+ "testing"
+)
+
+// Run a testing.M, first calling flag.Parse if not flag.Parsed.
+func Run(m *testing.M) {
+ if !flag.Parsed() {
+ flag.Parse()
+ }
+ os.Exit(m.Run())
+}
diff --git a/.test-infra/mock-apis/src/main/go/test/integration/vars.go
b/.test-infra/mock-apis/src/main/go/test/integration/vars.go
new file mode 100644
index 00000000000..92ba445fdfa
--- /dev/null
+++ b/.test-infra/mock-apis/src/main/go/test/integration/vars.go
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integration
+
+import (
+ "flag"
+ "fmt"
+)
+
+const (
+ grpcServiceEndpointFlag = "grpc_service_endpoint"
+ httpServiceEndpointFlag = "http_service_endpoint"
+
+ moreInfoUrl =
"https://github.com/apache/beam/tree/master/.test-infra/mock-apis#writing-integration-tests";
+)
+
+var (
+ moreInfo = fmt.Sprintf("See %s for more information on how to get the
relevant value for your test.", moreInfoUrl)
+
+ requiredFlags = []string{
+ grpcServiceEndpointFlag,
+ httpServiceEndpointFlag,
+ }
+)
+
+// The following flags apply to one or more integration tests and used via
+// go test ./src/main/go/test/integration/...
+var (
+ // GRPCServiceEndpoint is the address of the deployed service.
+ GRPCServiceEndpoint = flag.String(grpcServiceEndpointFlag, "",
+ "The endpoint to target gRPC calls to a service. "+moreInfo)
+
+ // HTTPServiceEndpoint is the address of the deployed service.
+ HTTPServiceEndpoint = flag.String(httpServiceEndpointFlag, "",
+ "The endpoint to target HTTP calls to a service. "+moreInfo)
+)
