This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new 9ba22ef Use pytest in integration test (#715)
9ba22ef is described below
commit 9ba22ef2a9bcc22864f4d6c1de90ecdc2fb4204b
Author: Jiayu Liu <[email protected]>
AuthorDate: Tue Jul 13 21:44:59 2021 +0800
Use pytest in integration test (#715)
* use pytest
* update to use pytest
---
.github/workflows/rust.yml | 4 +-
integration-tests/requirements.in | 18 ++++++
integration-tests/requirements.txt | 118 ++++++++++++++++++++++++++++++++++
integration-tests/test_psql_parity.py | 35 +++++-----
4 files changed, 153 insertions(+), 22 deletions(-)
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 4a994bf..2454d10 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -163,7 +163,7 @@ jobs:
- name: Install Python dependencies
run: |
python -m pip install --upgrade pip setuptools wheel
- python -m pip install --upgrade numpy==1.20.3 pandas==1.2.4
+ python -m pip install -r integration-tests/requirements.txt
- name: Allow access of psql
run: |
# make sure psql can access the server
@@ -194,7 +194,7 @@ jobs:
- name: Build datafusion-cli
run: cargo build --bin datafusion-cli
- name: Test Psql Parity
- run: python -m unittest -v integration-tests/test_psql_parity.py
+ run: python -m pytest -v integration-tests/test_psql_parity.py
env:
POSTGRES_HOST: localhost
POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }}
diff --git a/integration-tests/requirements.in
b/integration-tests/requirements.in
new file mode 100644
index 0000000..782329d
--- /dev/null
+++ b/integration-tests/requirements.in
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+pytest
+numpy
+pandas
diff --git a/integration-tests/requirements.txt
b/integration-tests/requirements.txt
new file mode 100644
index 0000000..3f8b113
--- /dev/null
+++ b/integration-tests/requirements.txt
@@ -0,0 +1,118 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This file is autogenerated by pip-compile with python 3.8
+# To update, run:
+#
+# pip-compile --generate-hashes
+#
+attrs==21.2.0 \
+
--hash=sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1 \
+
--hash=sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb
+ # via pytest
+iniconfig==1.1.1 \
+
--hash=sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3 \
+
--hash=sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32
+ # via pytest
+numpy==1.21.0 \
+
--hash=sha256:1a784e8ff7ea2a32e393cc53eb0003eca1597c7ca628227e34ce34eb11645a0e \
+
--hash=sha256:2ba579dde0563f47021dcd652253103d6fd66165b18011dce1a0609215b2791e \
+
--hash=sha256:3537b967b350ad17633b35c2f4b1a1bbd258c018910b518c30b48c8e41272717 \
+
--hash=sha256:3c40e6b860220ed862e8097b8f81c9af6d7405b723f4a7af24a267b46f90e461 \
+
--hash=sha256:598fe100b2948465cf3ed64b1a326424b5e4be2670552066e17dfaa67246011d \
+
--hash=sha256:620732f42259eb2c4642761bd324462a01cdd13dd111740ce3d344992dd8492f \
+
--hash=sha256:709884863def34d72b183d074d8ba5cfe042bc3ff8898f1ffad0209161caaa99 \
+
--hash=sha256:75579acbadbf74e3afd1153da6177f846212ea2a0cc77de53523ae02c9256513 \
+
--hash=sha256:7c55407f739f0bfcec67d0df49103f9333edc870061358ac8a8c9e37ea02fcd2 \
+
--hash=sha256:a1f2fb2da242568af0271455b89aee0f71e4e032086ee2b4c5098945d0e11cf6 \
+
--hash=sha256:a290989cd671cd0605e9c91a70e6df660f73ae87484218e8285c6522d29f6e38 \
+
--hash=sha256:ac4fd578322842dbda8d968e3962e9f22e862b6ec6e3378e7415625915e2da4d \
+
--hash=sha256:ad09f55cc95ed8d80d8ab2052f78cc21cb231764de73e229140d81ff49d8145e \
+
--hash=sha256:b9205711e5440954f861ceeea8f1b415d7dd15214add2e878b4d1cf2bcb1a914 \
+
--hash=sha256:bba474a87496d96e61461f7306fba2ebba127bed7836212c360f144d1e72ac54 \
+
--hash=sha256:bebab3eaf0641bba26039fb0b2c5bf9b99407924b53b1ea86e03c32c64ef5aef \
+
--hash=sha256:cc367c86eb87e5b7c9592935620f22d13b090c609f1b27e49600cd033b529f54 \
+
--hash=sha256:ccc6c650f8700ce1e3a77668bb7c43e45c20ac06ae00d22bdf6760b38958c883 \
+
--hash=sha256:cf680682ad0a3bef56dae200dbcbac2d57294a73e5b0f9864955e7dd7c2c2491 \
+
--hash=sha256:d2910d0a075caed95de1a605df00ee03b599de5419d0b95d55342e9a33ad1fb3 \
+
--hash=sha256:d5caa946a9f55511e76446e170bdad1d12d6b54e17a2afe7b189112ed4412bb8 \
+
--hash=sha256:d89b0dc7f005090e32bb4f9bf796e1dcca6b52243caf1803fdd2b748d8561f63 \
+
--hash=sha256:d95d16204cd51ff1a1c8d5f9958ce90ae190be81d348b514f9be39f878b8044a \
+
--hash=sha256:e4d5a86a5257843a18fb1220c5f1c199532bc5d24e849ed4b0289fb59fbd4d8f \
+
--hash=sha256:e58ddb53a7b4959932f5582ac455ff90dcb05fac3f8dcc8079498d43afbbde6c \
+
--hash=sha256:e80fe25cba41c124d04c662f33f6364909b985f2eb5998aaa5ae4b9587242cce \
+
--hash=sha256:eda2829af498946c59d8585a9fd74da3f810866e05f8df03a86f70079c7531dd \
+
--hash=sha256:fd0a359c1c17f00cb37de2969984a74320970e0ceef4808c32e00773b06649d9
+ # via
+ # -r requirements.in
+ # pandas
+packaging==21.0 \
+
--hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7 \
+
--hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14
+ # via pytest
+pandas==1.3.0 \
+
--hash=sha256:08eeff3da6a188e24db7f292b39a8ca9e073bf841fbbeadb946b3ad5c19d843e \
+
--hash=sha256:1ff13eed501e07e7fb26a4ea18a846b6e5d7de549b497025601fd9ccb7c1d123 \
+
--hash=sha256:522bfea92f3ef6207cadc7428bda1e7605dae0383b8065030e7b5d0266717b48 \
+
--hash=sha256:7897326cae660eee69d501cbfa950281a193fcf407393965e1bc07448e1cc35a \
+
--hash=sha256:798675317d0e4863a92a9a6bc5bd2490b5f6fef8c17b95f29e2e33f28bef9eca \
+
--hash=sha256:7d3cd2c99faa94d717ca00ea489264a291ad7209453dffbf059bfb7971fd3a61 \
+
--hash=sha256:823737830364d0e2af8c3912a28ba971296181a07950873492ed94e12d28c405 \
+
--hash=sha256:872aa91e0f9ca913046ab639d4181a899f5e592030d954d28c2529b88756a736 \
+
--hash=sha256:88864c1e28353b958b1f30e4193818519624ad9a1776921622a6a2a016d5d807 \
+
--hash=sha256:92835113a67cbd34747c198d41f09f4b63f6fe11ca5643baebc7ab1e30e89e95 \
+
--hash=sha256:98efc2d4983d5bb47662fe2d97b2c81b91566cb08b266490918b9c7d74a5ef64 \
+
--hash=sha256:b10d7910ae9d7920a5ff7816d794d99acbc361f7b16a0f017d4fa83ced8cb55e \
+
--hash=sha256:c554e6c9cf2d5ea1aba5979cc837b3649539ced0e18ece186f055450c86622e2 \
+
--hash=sha256:c746876cdd8380be0c3e70966d4566855901ac9aaa5e4b9ccaa5ca5311457d11 \
+
--hash=sha256:c81b8d91e9ae861eb4406b4e0f8d4dabbc105b9c479b3d1e921fba1d35b5b62a \
+
--hash=sha256:e6b75091fa54a53db3927b4d1bc997c23c5ba6f87acdfe1ee5a92c38c6b2ed6a \
+
--hash=sha256:ed4fc66f23fe17c93a5d439230ca2d6b5f8eac7154198d327dbe8a16d98f3f10 \
+
--hash=sha256:f058c786e7b0a9e7fa5e0b9f4422e0ccdd3bf3aa3053c18d77ed2a459bd9a45a \
+
--hash=sha256:fe7a549d10ca534797095586883a5c17d140d606747591258869c56e14d1b457
+ # via -r requirements.in
+pluggy==0.13.1 \
+
--hash=sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0 \
+
--hash=sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d
+ # via pytest
+py==1.10.0 \
+
--hash=sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3 \
+
--hash=sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a
+ # via pytest
+pyparsing==2.4.7 \
+
--hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \
+
--hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b
+ # via packaging
+pytest==6.2.4 \
+
--hash=sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b \
+
--hash=sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890
+ # via -r requirements.in
+python-dateutil==2.8.1 \
+
--hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
+
--hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a
+ # via pandas
+pytz==2021.1 \
+
--hash=sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da \
+
--hash=sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798
+ # via pandas
+six==1.16.0 \
+
--hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
+
--hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
+ # via python-dateutil
+toml==0.10.2 \
+
--hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
+
--hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f
+ # via pytest
diff --git a/integration-tests/test_psql_parity.py
b/integration-tests/test_psql_parity.py
index 39cfdee..a160d3e 100644
--- a/integration-tests/test_psql_parity.py
+++ b/integration-tests/test_psql_parity.py
@@ -14,13 +14,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import pandas as pd
-import numpy as np
import io
import os
import subprocess
from pathlib import Path
-import unittest
+
+import numpy as np
+import pandas as pd
+import pytest
pg_db, pg_user, pg_host, pg_port = [
os.environ.get(i)
@@ -70,22 +71,16 @@ def generate_csv_from_psql(fname: str):
)
-class PsqlParityTest(unittest.TestCase):
- def test_parity(self):
- root = Path(os.path.dirname(__file__)) / "sqls"
- files = set(root.glob("*.sql"))
- self.assertEqual(len(files), 14, msg="tests are missed")
- for fname in files:
- with self.subTest(fname=fname):
- datafusion_output = pd.read_csv(
- io.BytesIO(generate_csv_from_datafusion(fname))
- )
- psql_output =
pd.read_csv(io.BytesIO(generate_csv_from_psql(fname)))
- self.assertTrue(
- np.allclose(datafusion_output, psql_output,
equal_nan=True),
- msg=f"datafusion output=\n{datafusion_output},
psql_output=\n{psql_output}",
- )
+root = Path(os.path.dirname(__file__)) / "sqls"
+test_files = set(root.glob("*.sql"))
+
+class TestPsqlParity:
+ def test_tests_count(self):
+ assert len(test_files) == 14, "tests are missed"
-if __name__ == "__main__":
- unittest.main()
+ @pytest.mark.parametrize("fname", test_files)
+ def test_sql_file(self, fname):
+ datafusion_output =
pd.read_csv(io.BytesIO(generate_csv_from_datafusion(fname)))
+ psql_output = pd.read_csv(io.BytesIO(generate_csv_from_psql(fname)))
+ np.testing.assert_allclose(datafusion_output, psql_output,
equal_nan=True)
