This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 2d3a35007 Use http with fake-gcs (#2632)
2d3a35007 is described below
commit 2d3a350074dfb1d89a5d15ff8b37cae68eb8aa6f
Author: Raphael Taylor-Davies <[email protected]>
AuthorDate: Fri Sep 2 12:34:22 2022 +0100
Use http with fake-gcs (#2632)
---
.github/workflows/object_store.yml | 75 ++++++++++++++------------------------
object_store/src/gcp/mod.rs | 21 -----------
2 files changed, 27 insertions(+), 69 deletions(-)
diff --git a/.github/workflows/object_store.yml
b/.github/workflows/object_store.yml
index 118dbd932..6996aa706 100644
--- a/.github/workflows/object_store.yml
+++ b/.github/workflows/object_store.yml
@@ -60,74 +60,53 @@ jobs:
run: cargo clippy -p object_store --all-features --all-targets -- -D
warnings
# test the crate
+ # This runs outside a container to workaround lack of support for passing
arguments
+ # to service containers - https://github.com/orgs/community/discussions/26688
linux-test:
name: Emulator Tests
runs-on: ubuntu-latest
- services:
- fake-gcs:
- image: fsouza/fake-gcs-server
- ports:
- - 4443:4443
- localstack:
- image: localstack/localstack:0.14.4
- ports:
- - 4566:4566
- ec2-metadata:
- image: amazon/amazon-ec2-metadata-mock:v1.9.2
- ports:
- - 1338:1338
- env:
- # Only allow IMDSv2
- AEMM_IMDSV2: "1"
- azurite:
- image: mcr.microsoft.com/azure-storage/azurite
- ports:
- - 10000:10002
- container:
- image: amd64/rust
- env:
- # Disable full debug symbol generation to speed up CI build and keep
memory down
- # "1" means line tables only, which is useful for panic tracebacks.
- RUSTFLAGS: "-C debuginfo=1"
- # https://github.com/rust-lang/cargo/issues/10280
- CARGO_NET_GIT_FETCH_WITH_CLI: "true"
- RUST_BACKTRACE: "1"
- # Run integration tests
- TEST_INTEGRATION: 1
- EC2_METADATA_ENDPOINT: http://ec2-metadata:1338
- AZURE_USE_EMULATOR: "1"
- AZURITE_BLOB_STORAGE_URL: "http://azurite:10000";
- AZURITE_QUEUE_STORAGE_URL: "http://azurite:10001";
- GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json"
- OBJECT_STORE_BUCKET: test-bucket
+ env:
+ # Disable full debug symbol generation to speed up CI build and keep
memory down
+ # "1" means line tables only, which is useful for panic tracebacks.
+ RUSTFLAGS: "-C debuginfo=1"
+ # https://github.com/rust-lang/cargo/issues/10280
+ CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+ RUST_BACKTRACE: "1"
+ # Run integration tests
+ TEST_INTEGRATION: 1
+ EC2_METADATA_ENDPOINT: http://localhost:1338
+ AZURE_USE_EMULATOR: "1"
+ AZURITE_BLOB_STORAGE_URL: "http://localhost:10000";
+ AZURITE_QUEUE_STORAGE_URL: "http://localhost:10001";
+ GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json"
+ OBJECT_STORE_BUCKET: test-bucket
steps:
- uses: actions/checkout@v3
- name: Configure Fake GCS Server (GCP emulation)
run: |
- curl --insecure -v -X POST --data-binary '{"name":"test-bucket"}' -H
"Content-Type: application/json" "https://fake-gcs:4443/storage/v1/b";
- echo '{"gcs_base_url": "https://fake-gcs:4443";, "disable_oauth":
true, "client_email": "", "private_key": ""}' > "$GOOGLE_SERVICE_ACCOUNT"
+ docker run -d -p 4443:4443 fsouza/fake-gcs-server -scheme http
+ curl -v -X POST --data-binary '{"name":"test-bucket"}' -H
"Content-Type: application/json" "http://localhost:4443/storage/v1/b";
+ echo '{"gcs_base_url": "http://localhost:4443";, "disable_oauth":
true, "client_email": "", "private_key": ""}' > "$GOOGLE_SERVICE_ACCOUNT"
- name: Setup LocalStack (AWS emulation)
env:
AWS_DEFAULT_REGION: "us-east-1"
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
- AWS_ENDPOINT: http://localstack:4566
+ AWS_ENDPOINT: http://localhost:4566
run: |
- cd /tmp
- curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip"; -o
"awscliv2.zip"
- unzip awscliv2.zip
- ./aws/install
- aws --endpoint-url=http://localstack:4566 s3 mb s3://test-bucket
+ docker run -d -p 4566:4566 localstack/localstack:0.14.4
+ docker run -d -p 1338:1338 amazon/amazon-ec2-metadata-mock:v1.9.2
--imdsv2
+ aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket
- name: Configure Azurite (Azure emulation)
# the magical connection string is from
#
https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings
run: |
- curl -sL https://aka.ms/InstallAzureCLIDeb | bash
- az storage container create -n test-bucket --connection-string
'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;'
+ docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002
mcr.microsoft.com/azure-storage/azurite
+ az storage container create -n test-bucket --connection-string
'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;'
- name: Setup Rust toolchain
run: |
@@ -139,7 +118,7 @@ jobs:
OBJECT_STORE_AWS_DEFAULT_REGION: "us-east-1"
OBJECT_STORE_AWS_ACCESS_KEY_ID: test
OBJECT_STORE_AWS_SECRET_ACCESS_KEY: test
- OBJECT_STORE_AWS_ENDPOINT: http://localstack:4566
+ OBJECT_STORE_AWS_ENDPOINT: http://localhost:4566
run: |
# run tests
cargo test -p object_store --features=aws,azure,gcp
diff --git a/object_store/src/gcp/mod.rs b/object_store/src/gcp/mod.rs
index 65adf9128..0ef4d3564 100644
--- a/object_store/src/gcp/mod.rs
+++ b/object_store/src/gcp/mod.rs
@@ -782,20 +782,6 @@ impl GoogleCloudStorageBuilder {
self
}
- /// Use the specified http [`Client`] (defaults to [`Client::new`])
- ///
- /// This allows you to set custom client options such as allowing
- /// non secure connections or custom headers.
- ///
- /// NOTE: Currently only available in `test`s to facilitate
- /// testing, to avoid leaking details and preserve our ability to
- /// make changes to the implementation.
- #[cfg(test)]
- pub fn with_client(mut self, client: Client) -> Self {
- self.client = Some(client);
- self
- }
-
/// Configure a connection to Google Cloud Storage, returning a
/// new [`GoogleCloudStorage`] and consuming `self`
pub fn build(self) -> Result<GoogleCloudStorage> {
@@ -923,13 +909,6 @@ mod test {
env::var("GOOGLE_SERVICE_ACCOUNT")
.expect("already checked GOOGLE_SERVICE_ACCOUNT")
)
- .with_client(
- // ignore HTTPS errors in tests so we can use fake-gcs
server
- Client::builder()
- .danger_accept_invalid_certs(true)
- .build()
- .expect("Error creating http client for testing")
- )
}
}};
}
