This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch iotdb in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 4bf9c3e3661ee1755e308684f6f9489ec4655b07 Author: Haonan <[email protected]> AuthorDate: Fri Oct 18 09:58:01 2024 +0800 Support building python whl package by maven (#265) * fix python package * package whl by maven * finish build * update version * upload whl * try package * try package * try package * try fix win * try fix win * try fix win * try fix win * finish * fix verify * fix pkg missing libtsfile * fix pkg missing libtsfile --- .github/workflows/unit-test.yml | 8 +++++++ .gitignore | 6 +++++ pom.xml | 19 +++++----------- python/pom.xml | 24 +++++++++++++++++++- python/requirements.txt | 1 + python/setup.py | 49 +++++++++++++++++++++++------------------ 6 files changed, 70 insertions(+), 37 deletions(-) diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index ec0e7782..c3c10a0c 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -84,3 +84,11 @@ jobs: shell: bash run: | ./mvnw${{ steps.platform_suffix.outputs.platform_suffix }} -P with-java,with-cpp,with-python clean verify + + - name: Upload whl Artifact + if: ${{ matrix.java == '17'}} + uses: actions/upload-artifact@v4 + with: + name: tsfile-${{ runner.os }}-whl + path: python/dist/tsfile-*.whl + retention-days: 1 diff --git a/.gitignore b/.gitignore index c2c059c0..2600d5a7 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,9 @@ docs/pnpm-lock.yaml # python files python/build +python/dist +python/__pycache__ +python/tsfile.egg-info python/tsfile/__pycache__ python/tsfile/*so* python/tsfile/*dll* @@ -32,3 +35,6 @@ python/venv/* cpp/cmake-build-debug-mingw/ .vscode/settings.json +.vscode/ + +build/* diff --git a/pom.xml b/pom.xml index 314a995d..84523135 100644 --- a/pom.xml +++ b/pom.xml @@ -125,6 +125,10 @@ <!-- generated by cython--> <exclude>**/tsfile/tsfile_pywrapper.cpp</exclude> <exclude>**/venv/**</exclude> + <!-- generated by wheel--> + <exclude>**/tsfile.egg-info/**</exclude> + <!-- Exclude third_party--> + <exclude>**/third_party/**</exclude> </excludes> </configuration> </plugin> @@ -597,6 +601,7 @@ <profile> <id>with-python</id> <modules> + <module>cpp</module> <module>python</module> </modules> </profile> @@ -658,7 +663,6 @@ <cmake.generator>Unix Makefiles</cmake.generator> <python.venv.bin>venv/bin/</python.venv.bin> <python.exe.bin>python3</python.exe.bin> - <cython.compiler>unix</cython.compiler> </properties> </profile> <!-- Profile for linux amd64 (mainly AMD Processors) (Self-Enabling) --> @@ -677,7 +681,6 @@ <cmake.generator>Unix Makefiles</cmake.generator> <python.venv.bin>venv/bin/</python.venv.bin> <python.exe.bin>python3</python.exe.bin> - <cython.compiler>unix</cython.compiler> </properties> </profile> <!-- Profile for linux aarch64 (mainly ARM64 Processors) (Self-Enabling) --> @@ -696,7 +699,6 @@ <cmake.generator>Unix Makefiles</cmake.generator> <python.venv.bin>venv/bin/</python.venv.bin> <python.exe.bin>python3</python.exe.bin> - <cython.compiler>unix</cython.compiler> </properties> </profile> <!-- Profile for mac x86_64 (mainly Intel Processors) (Self-Enabling) --> @@ -714,7 +716,6 @@ <cmake.generator>Unix Makefiles</cmake.generator> <python.venv.bin>venv/bin/</python.venv.bin> <python.exe.bin>python3</python.exe.bin> - <cython.compiler>unix</cython.compiler> </properties> </profile> <!-- Profile for mac aarch64 (mainly new Mac Processors) (Self-Enabling) --> @@ -732,7 +733,6 @@ <cmake.generator>Unix Makefiles</cmake.generator> <python.venv.bin>venv/bin/</python.venv.bin> <python.exe.bin>python3</python.exe.bin> - <cython.compiler>unix</cython.compiler> </properties> </profile> <!-- profile for windows x86_64 (mainly Intel Processors) (Self-Enabling) --> @@ -748,11 +748,8 @@ <os.suffix>win</os.suffix> <os.classifier>windows-x86_64</os.classifier> <cmake.generator>MinGW Makefiles</cmake.generator> - <!-- The generated code relied on Boost and that relies on VS and can't be built with MinGW --> - <!--cmake.generator>Visual Studio 17 2022</cmake.generator--> <python.venv.bin>venv/Scripts/</python.venv.bin> <python.exe.bin>python</python.exe.bin> - <cython.compiler>mingw32</cython.compiler> </properties> </profile> <!-- profile for windows amd64 (mainly AMD Processors) (Self-Enabling) --> @@ -768,11 +765,8 @@ <os.suffix>win</os.suffix> <os.classifier>windows-amd64</os.classifier> <cmake.generator>MinGW Makefiles</cmake.generator> - <!-- The generated code relied on Boost and that relies on VS and can't be built with MinGW --> - <!--cmake.generator>Visual Studio 17 2022</cmake.generator--> <python.venv.bin>venv/Scripts/</python.venv.bin> <python.exe.bin>python</python.exe.bin> - <cython.compiler>mingw32</cython.compiler> </properties> </profile> <!-- profile for windows aarch64 (mainly VM on newer Mac) (Self-Enabling) --> @@ -788,11 +782,8 @@ <os.suffix>win</os.suffix> <os.classifier>windows-aarch64</os.classifier> <cmake.generator>MinGW Makefiles</cmake.generator> - <!-- The generated code relied on Boost and that relies on VS and can't be built with MinGW --> - <!--cmake.generator>Visual Studio 17 2022</cmake.generator--> <python.venv.bin>venv/Scripts/</python.venv.bin> <python.exe.bin>python</python.exe.bin> - <cython.compiler>mingw32</cython.compiler> </properties> </profile> <!-- Little helper profile that will disable running the cmake tests when the maven tests are being skipped --> diff --git a/python/pom.xml b/python/pom.xml index 0d0d9999..d80f1b81 100644 --- a/python/pom.xml +++ b/python/pom.xml @@ -103,7 +103,6 @@ <argument>setup.py</argument> <argument>build_ext</argument> <argument>--inplace</argument> - <argument>--compiler=${cython.compiler}</argument> </arguments> </configuration> </execution> @@ -136,6 +135,20 @@ </arguments> </configuration> </execution> + <execution> + <id>build-whl</id> + <phase>package</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>${python.venv.bin}${python.exe.bin}</executable> + <arguments> + <argument>setup.py</argument> + <argument>bdist_wheel</argument> + </arguments> + </configuration> + </execution> </executions> </plugin> <plugin> @@ -152,6 +165,15 @@ <include>*.cpp</include> </includes> </fileset> + <fileset> + <directory>tsfile.egg-info</directory> + </fileset> + <fileset> + <directory>build</directory> + </fileset> + <fileset> + <directory>dist</directory> + </fileset> </filesets> </configuration> </plugin> diff --git a/python/requirements.txt b/python/requirements.txt index 0c82a92b..1f579636 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -21,4 +21,5 @@ cython==3.0.10 numpy==1.26.4 pandas==2.2.2 setuptools==70.0.0 +wheel diff --git a/python/setup.py b/python/setup.py index bd438823..01120060 100644 --- a/python/setup.py +++ b/python/setup.py @@ -24,28 +24,32 @@ import platform import shutil import os +system = platform.system() -def copy_lib_files(system, source_dir, target_dir, suffix): +def copy_lib_files(source_dir, target_dir, suffix): lib_file_name = f"libtsfile.{suffix}" source = os.path.join(source_dir, lib_file_name) target = os.path.join(target_dir, lib_file_name) - shutil.copyfile(source, target) + + if os.path.exists(source): + shutil.copyfile(source, target) if system == "Linux": - link_name = os.path.join(target_dir, f"libtsfile.so") + link_name = os.path.join(target_dir, "libtsfile.so") if os.path.exists(link_name): os.remove(link_name) os.symlink(lib_file_name, link_name) - if system == "Darwin": - link_name = os.path.join(target_dir, f"libtsfile.dylib") + elif system == "Darwin": + link_name = os.path.join(target_dir, "libtsfile.dylib") if os.path.exists(link_name): os.remove(link_name) os.symlink(lib_file_name, link_name) def copy_header(source, target): - shutil.copyfile(source, target) + if os.path.exists(source): + shutil.copyfile(source, target) class BuildExt(build_ext): @@ -55,20 +59,18 @@ class BuildExt(build_ext): ext.include_dirs.append(numpy_include) super().build_extensions() + def finalize_options(self): + if platform.system() == "Windows": + self.compiler = 'mingw32' + super().finalize_options() + + +project_dir = os.path.dirname(os.path.abspath(__file__)) -project_dir = os.path.dirname(__file__) libtsfile_shard_dir = os.path.join(project_dir, "..", "cpp", "target", "build", "lib") libtsfile_dir = os.path.join(project_dir, "tsfile") include_dir = os.path.join(project_dir, "tsfile") -source_file = os.path.join(project_dir, "tsfile", "tsfile_pywrapper.pyx") - -if platform.system() == "Darwin": - copy_lib_files("Darwin", libtsfile_shard_dir, libtsfile_dir, "1.0.dylib") -elif platform.system() == "Linux": - copy_lib_files("Linux", libtsfile_shard_dir, libtsfile_dir, "so.1.0") -else: - copy_lib_files("Windows", libtsfile_shard_dir, libtsfile_dir, "dll") - +source_file = os.path.join("tsfile", "tsfile_pywrapper.pyx") source_include_dir = os.path.join( project_dir, "..", "cpp", "src", "cwrapper", "TsFile-cwrapper.h" @@ -76,6 +78,13 @@ source_include_dir = os.path.join( target_include_dir = os.path.join(project_dir, "tsfile", "TsFile-cwrapper.h") copy_header(source_include_dir, target_include_dir) +if system == "Darwin": + copy_lib_files(libtsfile_shard_dir, libtsfile_dir, "1.0.dylib") +elif system == "Linux": + copy_lib_files(libtsfile_shard_dir, libtsfile_dir, "so.1.0") +else: + copy_lib_files(libtsfile_shard_dir, libtsfile_dir, "dll") + ext_modules_tsfile = [ Extension( "tsfile.tsfile_pywrapper", @@ -91,7 +100,7 @@ ext_modules_tsfile = [ setup( name="tsfile", - version="0.1", + version="1.2.0.dev0", description="Tsfile reader and writer for python", url="https://tsfile.apache.org";, author='"Apache TsFile"', @@ -102,11 +111,7 @@ setup( include_dirs=[np.get_include()], package_data={ "tsfile": [ - os.path.join("*tsfile", "*.so*"), - os.path.join("*tsfile", "*.dylib"), - os.path.join("*tsfile", "*.pyd"), - os.path.join("*tsfile", "*.dll"), - os.path.join("tsfile", "tsfile.py"), + "libtsfile.*", ] }, include_package_data=True,
