This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fury.git
The following commit(s) were added to refs/heads/main by this push:
new 4f2b88d42 feat(ci): support building python on windows (#1885)
4f2b88d42 is described below
commit 4f2b88d427dd68b78db8090b5ac79baf844a4ab3
Author: Junduo Dong <[email protected]>
AuthorDate: Mon Jan 20 23:57:36 2025 +0800
feat(ci): support building python on windows (#1885)
## What does this PR do?
Support building python on Windows.
1. Use (git) bash to run python steps on Windows (MYSY, MINGW)
2. Add missing headers
3. Rename FuryLogLevel::ERROR to FuryLogLevel::ERR. I don't know why
this enum make build failed, but it is successed if renaming it to
FuryLogLevel::ERR
4. Rename pyx built dynamic lib name '*.so' to
['*.pyd'](https://docs.python.org/3.10/faq/windows.html#is-a-pyd-file-the-same-as-a-dll)
5. Lock bazel version to `6.3.2` on Windows
6. Unify Python type `Integer` memoryview formats (typecodes) on
different operating systems
([ref](https://docs.python.org/3/library/array.html))
| | int16 | int32 | int64 |
|-----------------|-------|-------|-------|
| Linux(64-bit) | 'h' | 'i' | 'l' |
| MacOS(64-bit) | 'h' | 'i' | 'l' |
| Windows(64-bit) | 'h' | 'l' | 'q' |
## Related issues
Close #798
## Does this PR introduce any user-facing change?
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
---------
Signed-off-by: Junduo Dong <[email protected]>
Co-authored-by: Shawn Yang <[email protected]>
---
.bazelversion | 1 +
.github/workflows/ci.yml | 12 +++-
.gitignore | 1 +
BUILD | 17 ++++--
ci/deploy.sh | 4 +-
ci/run_ci.sh | 6 ++
cpp/fury/thirdparty/MurmurHash3.cc | 1 +
cpp/fury/util/logging.cc | 4 +-
cpp/fury/util/logging.h | 2 +-
python/pyfury/_util.pyx | 10 +++
python/pyfury/serializer.py | 107 +++++++++++++++++++++++----------
python/pyfury/tests/test_serializer.py | 23 +++++++
12 files changed, 145 insertions(+), 43 deletions(-)
diff --git a/.bazelversion b/.bazelversion
new file mode 100644
index 000000000..f9da12e11
--- /dev/null
+++ b/.bazelversion
@@ -0,0 +1 @@
+6.3.2
\ No newline at end of file
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 72201cff9..0bb76ce1d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -215,10 +215,11 @@ jobs:
name: Python CI
# Fix python 3.6 install issue, see
#
https://github.com/rwth-i6/returnn/commit/38ecab17d781c4b74db6a174c8097187380b4ddc
- runs-on: ubuntu-20.04
+ runs-on: ${{ matrix.os }}
strategy:
matrix:
python-version: [3.8, 3.12]
+ os: [ubuntu-20.04, windows-2022]
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
@@ -226,8 +227,15 @@ jobs:
with:
python-version: ${{ matrix.python-version }}
- name: Install bazel
- run: ./ci/run_ci.sh install_bazel
+ shell: bash
+ run: |
+ if [ "$RUNNER_OS" == "Windows" ]; then
+ ./ci/run_ci.sh install_bazel_windows
+ else
+ ./ci/run_ci.sh install_bazel
+ fi
- name: Run Python CI
+ shell: bash
run: ./ci/run_ci.sh python
go:
diff --git a/.gitignore b/.gitignore
index 53cfc24de..f4faab088 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
**/*.cpp
**/*.so
**/*.dylib
+**/*.pyd
bazel-*
.whl
python/.cache
diff --git a/BUILD b/BUILD
index d5f106337..cc438bd3f 100644
--- a/BUILD
+++ b/BUILD
@@ -103,10 +103,19 @@ genrule(
set -e
set -x
WORK_DIR=$$(pwd)
- cp -f $(location python/pyfury/_util.so) "$$WORK_DIR/python/pyfury"
- cp -f $(location python/pyfury/lib/mmh3/mmh3.so)
"$$WORK_DIR/python/pyfury/lib/mmh3"
- cp -f $(location python/pyfury/format/_format.so)
"$$WORK_DIR/python/pyfury/format"
- cp -f $(location python/pyfury/_serialization.so)
"$$WORK_DIR/python/pyfury"
+ u_name=`uname -s`
+ if [ "$${u_name: 0: 4}" == "MING" ] || [ "$${u_name: 0: 4}" == "MSYS" ]
+ then
+ cp -f $(location python/pyfury/_util.so)
"$$WORK_DIR/python/pyfury/_util.pyd"
+ cp -f $(location python/pyfury/lib/mmh3/mmh3.so)
"$$WORK_DIR/python/pyfury/lib/mmh3/mmh3.pyd"
+ cp -f $(location python/pyfury/format/_format.so)
"$$WORK_DIR/python/pyfury/format/_format.pyd"
+ cp -f $(location python/pyfury/_serialization.so)
"$$WORK_DIR/python/pyfury/_serialization.pyd"
+ else
+ cp -f $(location python/pyfury/_util.so) "$$WORK_DIR/python/pyfury"
+ cp -f $(location python/pyfury/lib/mmh3/mmh3.so)
"$$WORK_DIR/python/pyfury/lib/mmh3"
+ cp -f $(location python/pyfury/format/_format.so)
"$$WORK_DIR/python/pyfury/format"
+ cp -f $(location python/pyfury/_serialization.so)
"$$WORK_DIR/python/pyfury"
+ fi
echo $$(date) > $@
""",
local = 1,
diff --git a/ci/deploy.sh b/ci/deploy.sh
index e514848c0..4a8ab181b 100755
--- a/ci/deploy.sh
+++ b/ci/deploy.sh
@@ -47,9 +47,8 @@ VERSIONS=("3.7"
"3.11"
"3.12")
-source $(conda info --base)/etc/profile.d/conda.sh
-
create_py_envs() {
+ source $(conda info --base)/etc/profile.d/conda.sh
for version in "${VERSIONS[@]}"; do
conda create -y --name "py$version" python="$version"
done
@@ -94,6 +93,7 @@ deploy_jars() {
}
deploy_python() {
+ source $(conda info --base)/etc/profile.d/conda.sh
if command -v pyenv; then
pyenv local system
fi
diff --git a/ci/run_ci.sh b/ci/run_ci.sh
index 14664e9bb..e56204417 100755
--- a/ci/run_ci.sh
+++ b/ci/run_ci.sh
@@ -78,6 +78,12 @@ install_bazel() {
fi
}
+install_bazel_windows() {
+ choco install bazel --version=6.3.2 --force
+ VERSION=`bazel version`
+ echo "bazel version: $VERSION"
+}
+
JDKS=(
"zulu21.28.85-ca-jdk21.0.0-linux_x64"
"zulu17.44.17-ca-crac-jdk17.0.8-linux_x64"
diff --git a/cpp/fury/thirdparty/MurmurHash3.cc
b/cpp/fury/thirdparty/MurmurHash3.cc
index c9d144692..7267187cc 100644
--- a/cpp/fury/thirdparty/MurmurHash3.cc
+++ b/cpp/fury/thirdparty/MurmurHash3.cc
@@ -19,6 +19,7 @@
#define FORCE_INLINE __forceinline
#include <cstdint>
+#include <cstdlib>
#define ROTL32(x, y) _rotl(x, y)
#define ROTL64(x, y) _rotl64(x, y)
diff --git a/cpp/fury/util/logging.cc b/cpp/fury/util/logging.cc
index c52d5feb7..9718a31be 100644
--- a/cpp/fury/util/logging.cc
+++ b/cpp/fury/util/logging.cc
@@ -52,7 +52,7 @@ std::string GetCallTrace() {
std::unordered_map<FuryLogLevel, std::string> log_level_to_str = {
{FuryLogLevel::DEBUG, "DEBUG"}, {FuryLogLevel::INFO, "INFO"},
- {FuryLogLevel::WARNING, "WARNING"}, {FuryLogLevel::ERROR, "ERROR"},
+ {FuryLogLevel::WARNING, "WARNING"}, {FuryLogLevel::ERR, "ERROR"},
{FuryLogLevel::FATAL, "FATAL"},
};
@@ -77,7 +77,7 @@ FuryLogLevel FuryLog::GetLogLevel() {
} else if (data == "warning") {
severity_threshold = FuryLogLevel::WARNING;
} else if (data == "error") {
- severity_threshold = FuryLogLevel::ERROR;
+ severity_threshold = FuryLogLevel::ERR;
} else if (data == "fatal") {
severity_threshold = FuryLogLevel::FATAL;
} else {
diff --git a/cpp/fury/util/logging.h b/cpp/fury/util/logging.h
index 1770e3feb..addb21027 100644
--- a/cpp/fury/util/logging.h
+++ b/cpp/fury/util/logging.h
@@ -35,7 +35,7 @@ enum class FuryLogLevel {
DEBUG = -1,
INFO = 0,
WARNING = 1,
- ERROR = 2,
+ ERR = 2,
FATAL = 3
};
diff --git a/python/pyfury/_util.pyx b/python/pyfury/_util.pyx
index 3d0ac05fd..de4eab33c 100644
--- a/python/pyfury/_util.pyx
+++ b/python/pyfury/_util.pyx
@@ -29,10 +29,13 @@ from libcpp cimport bool as c_bool
from pyfury.includes.libutil cimport(
CBuffer, AllocateBuffer, GetBit, SetBit, ClearBit, SetBitTo, CStatus,
StatusCode, utf16HasSurrogatePairs
)
+import os
cdef int32_t max_buffer_size = 2 ** 31 - 1
cdef int UTF16_LE = -1
+cdef c_bool _WINDOWS = os.name == 'nt'
+
@cython.final
cdef class Buffer:
@@ -686,6 +689,13 @@ cdef inline uint8_t* get_address(v):
signed_int_data = v
ptr = <uint8_t*>(&signed_int_data[0])
elif dtype == "l":
+ if _WINDOWS:
+ signed_int_data = v
+ ptr = <uint8_t*>(&signed_int_data[0])
+ else:
+ signed_long_data = v
+ ptr = <uint8_t*>(&signed_long_data[0])
+ elif dtype == "q":
signed_long_data = v
ptr = <uint8_t*>(&signed_long_data[0])
elif dtype == "f":
diff --git a/python/pyfury/serializer.py b/python/pyfury/serializer.py
index f7276f7d8..b42042399 100644
--- a/python/pyfury/serializer.py
+++ b/python/pyfury/serializer.py
@@ -43,6 +43,8 @@ from pyfury._fury import (
BufferObject,
)
+_WINDOWS = os.name == "nt"
+
from pyfury._serialization import ENABLE_FURY_CYTHON_SERIALIZATION
if ENABLE_FURY_CYTHON_SERIALIZATION:
@@ -420,33 +422,63 @@ class DataClassSerializer(Serializer):
# Use numpy array or python array module.
-typecode_dict = {
- # use bytes serializer for byte array.
- "h": (2, Int16ArrayType, TypeId.INT16_ARRAY),
- "i": (4, Int32ArrayType, TypeId.INT32_ARRAY),
- "l": (8, Int64ArrayType, TypeId.INT64_ARRAY),
- "f": (4, Float32ArrayType, TypeId.FLOAT32_ARRAY),
- "d": (8, Float64ArrayType, TypeId.FLOAT64_ARRAY),
-}
-
-typeid_code = {
- TypeId.INT16_ARRAY: "h",
- TypeId.INT32_ARRAY: "i",
- TypeId.INT64_ARRAY: "l",
- TypeId.FLOAT32_ARRAY: "f",
- TypeId.FLOAT64_ARRAY: "d",
-}
+typecode_dict = (
+ {
+ # use bytes serializer for byte array.
+ "h": (2, Int16ArrayType, TypeId.INT16_ARRAY),
+ "i": (4, Int32ArrayType, TypeId.INT32_ARRAY),
+ "l": (8, Int64ArrayType, TypeId.INT64_ARRAY),
+ "f": (4, Float32ArrayType, TypeId.FLOAT32_ARRAY),
+ "d": (8, Float64ArrayType, TypeId.FLOAT64_ARRAY),
+ }
+ if not _WINDOWS
+ else {
+ "h": (2, Int16ArrayType, TypeId.INT16_ARRAY),
+ "l": (4, Int32ArrayType, TypeId.INT32_ARRAY),
+ "q": (8, Int64ArrayType, TypeId.INT64_ARRAY),
+ "f": (4, Float32ArrayType, TypeId.FLOAT32_ARRAY),
+ "d": (8, Float64ArrayType, TypeId.FLOAT64_ARRAY),
+ }
+)
+
+typeid_code = (
+ {
+ TypeId.INT16_ARRAY: "h",
+ TypeId.INT32_ARRAY: "i",
+ TypeId.INT64_ARRAY: "l",
+ TypeId.FLOAT32_ARRAY: "f",
+ TypeId.FLOAT64_ARRAY: "d",
+ }
+ if not _WINDOWS
+ else {
+ TypeId.INT16_ARRAY: "h",
+ TypeId.INT32_ARRAY: "l",
+ TypeId.INT64_ARRAY: "q",
+ TypeId.FLOAT32_ARRAY: "f",
+ TypeId.FLOAT64_ARRAY: "d",
+ }
+)
class PyArraySerializer(CrossLanguageCompatibleSerializer):
typecode_dict = typecode_dict
- typecodearray_type = {
- "h": Int16ArrayType,
- "i": Int32ArrayType,
- "l": Int64ArrayType,
- "f": Float32ArrayType,
- "d": Float64ArrayType,
- }
+ typecodearray_type = (
+ {
+ "h": Int16ArrayType,
+ "i": Int32ArrayType,
+ "l": Int64ArrayType,
+ "f": Float32ArrayType,
+ "d": Float64ArrayType,
+ }
+ if not _WINDOWS
+ else {
+ "h": Int16ArrayType,
+ "l": Int32ArrayType,
+ "q": Int64ArrayType,
+ "f": Float32ArrayType,
+ "d": Float64ArrayType,
+ }
+ )
def __init__(self, fury, ftype, type_id: str):
super().__init__(fury, ftype)
@@ -511,15 +543,26 @@ class DynamicPyArraySerializer(Serializer):
if np:
- _np_dtypes_dict = {
- # use bytes serializer for byte array.
- np.dtype(np.bool_): (1, "?", BoolNDArrayType, TypeId.BOOL_ARRAY),
- np.dtype(np.int16): (2, "h", Int16NDArrayType, TypeId.INT16_ARRAY),
- np.dtype(np.int32): (4, "i", Int32NDArrayType, TypeId.INT32_ARRAY),
- np.dtype(np.int64): (8, "l", Int64NDArrayType, TypeId.INT64_ARRAY),
- np.dtype(np.float32): (4, "f", Float32NDArrayType,
TypeId.FLOAT32_ARRAY),
- np.dtype(np.float64): (8, "d", Float64NDArrayType,
TypeId.FLOAT64_ARRAY),
- }
+ _np_dtypes_dict = (
+ {
+ # use bytes serializer for byte array.
+ np.dtype(np.bool_): (1, "?", BoolNDArrayType, TypeId.BOOL_ARRAY),
+ np.dtype(np.int16): (2, "h", Int16NDArrayType, TypeId.INT16_ARRAY),
+ np.dtype(np.int32): (4, "i", Int32NDArrayType, TypeId.INT32_ARRAY),
+ np.dtype(np.int64): (8, "l", Int64NDArrayType, TypeId.INT64_ARRAY),
+ np.dtype(np.float32): (4, "f", Float32NDArrayType,
TypeId.FLOAT32_ARRAY),
+ np.dtype(np.float64): (8, "d", Float64NDArrayType,
TypeId.FLOAT64_ARRAY),
+ }
+ if not _WINDOWS
+ else {
+ np.dtype(np.bool_): (1, "?", BoolNDArrayType, TypeId.BOOL_ARRAY),
+ np.dtype(np.int16): (2, "h", Int16NDArrayType, TypeId.INT16_ARRAY),
+ np.dtype(np.int32): (4, "l", Int32NDArrayType, TypeId.INT32_ARRAY),
+ np.dtype(np.int64): (8, "q", Int64NDArrayType, TypeId.INT64_ARRAY),
+ np.dtype(np.float32): (4, "f", Float32NDArrayType,
TypeId.FLOAT32_ARRAY),
+ np.dtype(np.float64): (8, "d", Float64NDArrayType,
TypeId.FLOAT64_ARRAY),
+ }
+ )
else:
_np_dtypes_dict = {}
diff --git a/python/pyfury/tests/test_serializer.py
b/python/pyfury/tests/test_serializer.py
index ec6d399d4..923058095 100644
--- a/python/pyfury/tests/test_serializer.py
+++ b/python/pyfury/tests/test_serializer.py
@@ -19,6 +19,7 @@ import array
import datetime
import gc
import io
+import os
import pickle
import weakref
from enum import Enum
@@ -249,6 +250,28 @@ def test_array_serializer(language):
np.testing.assert_array_equal(new_arr, arr)
+def test_numpy_array_memoryview():
+ _WINDOWS = os.name == "nt"
+ if _WINDOWS:
+ arr = np.array(list(range(10)), dtype="int32")
+ view = memoryview(arr)
+ assert view.format == "l"
+ assert view.itemsize == 4
+ arr = np.array(list(range(10)), dtype="int64")
+ view = memoryview(arr)
+ assert view.format == "q"
+ assert view.itemsize == 8
+ else:
+ arr = np.array(list(range(10)), dtype="int32")
+ view = memoryview(arr)
+ assert view.format == "i"
+ assert view.itemsize == 4
+ arr = np.array(list(range(10)), dtype="int64")
+ view = memoryview(arr)
+ assert view.format == "l"
+ assert view.itemsize == 8
+
+
def ser_de(fury, obj):
binary = fury.serialize(obj)
return fury.deserialize(binary)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
