From 0621b959de0c25020418d37e7831fe59fc74e1f3 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 27 Aug 2024 18:29:25 +0000 Subject: [PATCH 01/33] Testing windows - self.arch to use lowercase as we expect cibuildwheel to pass amd64 while platform.machine() returns AMD64 on windows - Added Windows and amd64 as supported (list check) - Added archive suffix and archive format to support ZIP archive - Custom rename of arch in filename as openZIM uses x86_64 - Copying dll from bin/ folder (tree is different) and .lib file (not sure if needed) - Cleaning dll/lib as well - Setting to the exact nightly date we have it for (not targetting lastest nightly as it requires addtional ZIP handling TBD) - forcing cibuildwheel to only do windows py312 on amd64 for tests speed - forcing those wheels on windows branch --- .github/workflows/test.yml | 2 +- .github/workflows/wheels.yml | 3 ++- pyproject.toml | 2 +- setup.py | 51 ++++++++++++++++++++++++++---------- 4 files changed, 41 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cb7189f..b08e7dc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: test on: [push] env: - LIBZIM_DL_VERSION: "9.1.0" + LIBZIM_DL_VERSION: "2024-08-27" MACOSX_DEPLOYMENT_TARGET: "12.0" jobs: diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 6c276ac..65ba97c 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -5,9 +5,10 @@ on: push: branches: - main + - windows env: - LIBZIM_DL_VERSION: "9.1.0" + LIBZIM_DL_VERSION: "2024-08-27" MACOSX_DEPLOYMENT_TARGET: "12.0" CIBW_ENVIRONMENT_PASS_LINUX: "LIBZIM_DL_VERSION" CIBW_BUILD_VERBOSITY: "3" diff --git a/pyproject.toml b/pyproject.toml index 6614889..866d057 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ testpaths = ["tests"] pythonpath = ["."] [tool.cibuildwheel] -build = "*" +build = "cp312-win_amd64" # disabling windows until hhttps://github.com/kiwix/kiwix-build/issues/466 # disabling PyPy due to 2 failing tests skip = "pp* *-win*" diff --git a/setup.py b/setup.py index c32e191..fbeb0be 100755 --- a/setup.py +++ b/setup.py @@ -46,6 +46,7 @@ class Config: "Darwin": ["x86_64", "arm64"], "Linux": ["x86_64", "aarch64"], "Linux-musl": ["x86_64", "aarch64"], + "Windows": ["amd64"], } base_dir: pathlib.Path = Path(__file__).parent @@ -101,7 +102,7 @@ def arch(self) -> str: # we extract the cross-compile arch from it return ( os.getenv("_PYTHON_HOST_PLATFORM", "").rsplit("-", 1)[-1] - or sysplatform.machine() + or sysplatform.machine().lower() ) def check_platform(self): @@ -121,8 +122,19 @@ def libzim_fname(self): return { "Darwin": f"libzim.{self.libzim_major}.dylib", "Linux": f"libzim.so.{self.libzim_major}", + "Windows": f"zim-{self.libzim_major}.dll", }[self.platform] + @property + def archive_suffix(self): + if self.platform == "Windows": + return ".zip" + return ".tar.gz" + + @property + def archive_format(self): + return {".zip": "zip", ".tar.gz": "gztar"}.get(self.archive_suffix) + @property def is_musl(self) -> bool: """whether running on a musl system (Alpine)""" @@ -145,7 +157,13 @@ def get_download_filename(self, arch: Optional[str] = None) -> str: """filename to download to get binary libzim for platform/arch""" arch = arch or self.arch - lzplatform = {"Darwin": "macos", "Linux": "linux"}.get(self.platform) + # believe this is incorrect naming at openZIM ; will open ticket + if self.platform == "Windows" and arch == "amd64": + arch = "x86_64" + + lzplatform = {"Darwin": "macos", "Linux": "linux", "Windows": "win"}.get( + self.platform + ) variant = "" if self.platform == "Linux": @@ -157,7 +175,7 @@ def get_download_filename(self, arch: Optional[str] = None) -> str: version_suffix = f"-{self.libzim_dl_version}" return pathlib.Path( - f"libzim_{lzplatform}-{arch}{variant}{version_suffix}.tar.gz" + f"libzim_{lzplatform}-{arch}{variant}{version_suffix}{self.archive_suffix}" ).name def download_to_dest(self): @@ -227,15 +245,16 @@ def _download_and_extract(self, filename: str) -> pathlib.Path: print("> extracting archive") # extract into current folder (all files are inside an in-tar folder) - shutil.unpack_archive(fpath, self.base_dir, "gztar") + shutil.unpack_archive(fpath, self.base_dir, self.archive_format) # nightly have different download name and extracted folder name as it # uses a redirect + # TODO: FIX for zip if self.is_latest_nightly: tar = tarfile.open(fpath) folder = pathlib.Path(pathlib.Path(tar.firstmember.name).parts[0]) else: - folder = fpath.with_name(fpath.name.replace(".tar.gz", "")) + folder = fpath.with_name(fpath.name.replace(self.archive_suffix, "")) return folder @@ -254,6 +273,14 @@ def _install_from(self, folder: pathlib.Path): for fpath in folder.joinpath("lib").rglob("libzim.*"): print(f"{fpath} -> {libzim_dir / fpath.name}") os.replace(fpath, libzim_dir / fpath.name) + # windows has different folder and name + for fpath in folder.joinpath("bin").rglob("zim-*.dll"): + print(f"{fpath} -> {libzim_dir / fpath.name}") + os.replace(fpath, libzim_dir / fpath.name) + # windows again, not sure its required at all + for fpath in folder.joinpath("lib").rglob("zim.lib"): + print(f"{fpath} -> {libzim_dir / fpath.name}") + os.replace(fpath, libzim_dir / fpath.name) # remove temp folder shutil.rmtree(folder, ignore_errors=True) @@ -284,7 +311,7 @@ def cleanup(self): # we downloaded libzim, so we must remove it if self.download_libzim: print("removing downloaded libraries") - for fpath in self.dylib_file.parent.glob("*.[dylib|so]*"): + for fpath in self.dylib_file.parent.glob("*.[dylib|so|dll|lib]*"): print(">", fpath) fpath.unlink(missing_ok=True) if self.header_file.parent.exists(): @@ -490,11 +517,9 @@ class DownloadLibzim(Command): user_options = [] - def initialize_options(self): - ... + def initialize_options(self): ... - def finalize_options(self): - ... + def finalize_options(self): ... def run(self): config.download_to_dest() @@ -503,11 +528,9 @@ def run(self): class LibzimClean(Command): user_options = [] - def initialize_options(self): - ... + def initialize_options(self): ... - def finalize_options(self): - ... + def finalize_options(self): ... def run(self): config.cleanup() From f7a5b3e121a57d6ea89538bfeb0544ef3d8b2b91 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 27 Aug 2024 18:33:13 +0000 Subject: [PATCH 02/33] only run on windows --- .github/workflows/test.yml | 4 ++-- .github/workflows/wheels.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b08e7dc..5c75a6e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,8 +27,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04, macos-13] - python: ["3.8", "3.9", "3.10", "3.11", "3.12"] + os: [windows-latest] + python: ["3.12"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 65ba97c..294e703 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -20,7 +20,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, macos-13] # windows-2019 + os: [windows-latest] steps: - uses: actions/checkout@v3 From 96e4094a5ccf551ebf567447e95f03647b9d1327 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 27 Aug 2024 18:34:47 +0000 Subject: [PATCH 03/33] dont skip what we want --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 866d057..0eb3096 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ pythonpath = ["."] build = "cp312-win_amd64" # disabling windows until hhttps://github.com/kiwix/kiwix-build/issues/466 # disabling PyPy due to 2 failing tests -skip = "pp* *-win*" +# skip = "pp* *-win*" test-requires = ["pytest"] test-command = "py.test {project}/tests/" From da55f70588378e60591749fb585f34beb8c2d105 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Wed, 28 Aug 2024 10:12:34 +0000 Subject: [PATCH 04/33] ZIP needs a target folder and include folder needs to exist --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index fbeb0be..30fc891 100755 --- a/setup.py +++ b/setup.py @@ -244,8 +244,6 @@ def _download_and_extract(self, filename: str) -> pathlib.Path: print(f"> reusing local file {fpath}") print("> extracting archive") - # extract into current folder (all files are inside an in-tar folder) - shutil.unpack_archive(fpath, self.base_dir, self.archive_format) # nightly have different download name and extracted folder name as it # uses a redirect @@ -255,6 +253,9 @@ def _download_and_extract(self, filename: str) -> pathlib.Path: folder = pathlib.Path(pathlib.Path(tar.firstmember.name).parts[0]) else: folder = fpath.with_name(fpath.name.replace(self.archive_suffix, "")) + # unless for ZIP, extract to current folder (all files inside an in-tar folder) + extract_to = folder if self.archive_format == "zip" else self.base_dir + shutil.unpack_archive(fpath, extract_to, self.archive_format) return folder @@ -267,6 +268,7 @@ def _install_from(self, folder: pathlib.Path): shutil.rmtree(self.base_dir / "include" / "zim", ignore_errors=True) # copy new zim headers + (self.base_dir / "include").mkdir(exist_ok=True, parents=True) shutil.move(folder / "include" / "zim", self.base_dir / "include" / "zim") # copy new libs From 619f0e6455ccd9dd425859b27fcc2c5a2ba93183 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Wed, 28 Aug 2024 10:21:35 +0000 Subject: [PATCH 05/33] -Wextra is for gcc, not for windows --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 30fc891..7440ec2 100755 --- a/setup.py +++ b/setup.py @@ -387,6 +387,10 @@ def get_cython_extension(): else ["$ORIGIN/libzim/"] ) + extra_compile_args = ["-std=c++11", "-Wall"] + if config.platform != "Windows": + extra_compile_args.append("-Wextra") + wrapper_extension = Extension( name="libzim", sources=["libzim/libzim.pyx", "libzim/libwrapper.cpp"], @@ -394,7 +398,7 @@ def get_cython_extension(): libraries=["zim"], library_dirs=library_dirs, runtime_library_dirs=runtime_library_dirs, - extra_compile_args=["-std=c++11", "-Wall", "-Wextra"], + extra_compile_args=extra_compile_args, language="c++", define_macros=define_macros, ) From 85592756d7baec290a3da66193276e33dd3e4d28 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Fri, 30 Aug 2024 18:09:45 +0000 Subject: [PATCH 06/33] use release 9.2.3-1 (non-debug) --- .github/workflows/release.yaml | 2 +- .github/workflows/test.yml | 2 +- .github/workflows/wheels.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 10cc262..908730a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -6,7 +6,7 @@ on: - published env: - LIBZIM_DL_VERSION: "9.1.0" + LIBZIM_DL_VERSION: "9.2.3-1" MACOSX_DEPLOYMENT_TARGET: "12.0" CIBW_ENVIRONMENT_PASS_LINUX: "LIBZIM_DL_VERSION" # APPLE_SIGNING_KEYCHAIN_PATH set in prepare keychain step diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5c75a6e..9c2f800 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: test on: [push] env: - LIBZIM_DL_VERSION: "2024-08-27" + LIBZIM_DL_VERSION: "9.2.3-1" MACOSX_DEPLOYMENT_TARGET: "12.0" jobs: diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 294e703..aec86c5 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -8,7 +8,7 @@ on: - windows env: - LIBZIM_DL_VERSION: "2024-08-27" + LIBZIM_DL_VERSION: "9.2.3-1" MACOSX_DEPLOYMENT_TARGET: "12.0" CIBW_ENVIRONMENT_PASS_LINUX: "LIBZIM_DL_VERSION" CIBW_BUILD_VERBOSITY: "3" From ccaab6444db9fd784ac06c28a9547066d8d0e125 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Fri, 30 Aug 2024 19:06:34 +0000 Subject: [PATCH 07/33] Windows runtime library_path alternative Windows has no runtime_library path that's practical for us - require user to set PATH specificaly before running python - require user to use os.add_dll_directory() before `import python` - require user to load libzim DLL (via `ctypes.CDLL()`) in memory before `import python` - require us to change our API to use of of those trick before importing wrapper Most practical solution found was thus to install our companion DLLs (libzim and libicu) next to the wrapper one. This is done via a new command that edits the built wheel Added conditional support for _DEBUG as this is required to link against debug libzim DLLs --- MANIFEST.in | 1 + pyproject.toml | 5 ++- setup.cfg | 6 +++ setup.py | 86 +++++++++++++++++++++++++++++------- tests/test_libzim_creator.py | 2 +- 5 files changed, 82 insertions(+), 18 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 8c4a59c..142d275 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -15,3 +15,4 @@ exclude *.egg-info/* exclude libzim/*.dylib exclude libzim/*.so exclude libzim/*.so.* +exclude libzim/*.dll diff --git a/pyproject.toml b/pyproject.toml index 0eb3096..4eb6c52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = [ "setuptools == 68.2.2", "wheel == 0.41.3", "cython == 3.0.5" ] +requires = [ "setuptools == 68.2.2", "wheel == 0.41.3", "cython == 3.0.5", "delocate==0.12.0" ] build-backend = "setuptools.build_meta" [tool.black] @@ -24,6 +24,9 @@ manylinux-aarch64-image = "manylinux_2_28" manylinux-pypy_x86_64-image = "manylinux_2_28" manylinux-pypy_aarch64-image = "manylinux_2_28" +[tool.cibuildwheel.windows] +repair-wheel-command = "python.exe setup.py repair_win_wheel --destdir={dest_dir} --wheel={wheel}" + [tool.cibuildwheel.linux] archs = ["x86_64", "aarch64"] diff --git a/setup.cfg b/setup.cfg index d38e1f8..76fab60 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,6 +50,12 @@ test_requires = libzim = libzim.9.dylib libzim.so.9 + zim-9.dll + icuuc74.dll + icutu74.dll + icuio74.dll + icuin74.dll + icudt74.dll [isort] profile = black diff --git a/setup.py b/setup.py index 7440ec2..af6268a 100755 --- a/setup.py +++ b/setup.py @@ -9,6 +9,8 @@ The Cython and Cythonize compilation is done automatically by the build backend""" +from __future__ import annotations + import os import pathlib import platform as sysplatform @@ -21,10 +23,10 @@ import urllib.request from ctypes.util import find_library from pathlib import Path -from typing import Optional, Tuple from Cython.Build import cythonize from Cython.Distutils.build_ext import new_build_ext as build_ext +from delocate.wheeltools import InWheel from setuptools import Command, Extension, setup @@ -42,7 +44,10 @@ class Config: apple_signing_keychain: str = os.getenv("APPLE_SIGNING_KEYCHAIN_PATH") apple_signing_keychain_profile: str = os.getenv("APPLE_SIGNING_KEYCHAIN_PROFILE") - supported_platforms = { + # windows + _msvc_debug: bool = bool(os.getenv("MSVC_DEBUG")) + + supported_platforms = { # noqa: RUF012 "Darwin": ["x86_64", "arm64"], "Linux": ["x86_64", "aarch64"], "Linux-musl": ["x86_64", "aarch64"], @@ -52,8 +57,9 @@ class Config: base_dir: pathlib.Path = Path(__file__).parent # Avoid running cythonize on `setup.py clean` and similar - buildless_commands: Tuple[str] = ( + buildless_commands: tuple[str, ...] = ( "clean", + "repair_win_wheel", "--help", "egg_info", "--version", @@ -153,7 +159,15 @@ def wants_universal(self) -> bool: "universal2" ) - def get_download_filename(self, arch: Optional[str] = None) -> str: + @property + def use_msvc_debug(self) -> bool: + """whether to add _DEBUG define to compilation + + requires having python debug binaries installed. + mandatory for compiling against libzim nighlies""" + return self._msvc_debug or self.is_nightly + + def get_download_filename(self, arch: str | None = None) -> str: """filename to download to get binary libzim for platform/arch""" arch = arch or self.arch @@ -276,11 +290,11 @@ def _install_from(self, folder: pathlib.Path): print(f"{fpath} -> {libzim_dir / fpath.name}") os.replace(fpath, libzim_dir / fpath.name) # windows has different folder and name - for fpath in folder.joinpath("bin").rglob("zim-*.dll"): - print(f"{fpath} -> {libzim_dir / fpath.name}") - os.replace(fpath, libzim_dir / fpath.name) - # windows again, not sure its required at all - for fpath in folder.joinpath("lib").rglob("zim.lib"): + for fpath in ( + list(folder.joinpath("bin").rglob("zim-*.dll")) + + list(folder.joinpath("bin").rglob("icu*.dll")) + + list(folder.joinpath("lib").rglob("zim.lib")) + ): print(f"{fpath} -> {libzim_dir / fpath.name}") os.replace(fpath, libzim_dir / fpath.name) @@ -320,6 +334,18 @@ def cleanup(self): print("removing downloaded headers") shutil.rmtree(self.header_file.parent, ignore_errors=True) + def repair_windows_wheel(self, wheel: Path, dest_dir: Path): + """opens windows wheels in target folder and moves all DLLs files inside + subdirectories of the wheel to the root one (where wrapper is expected)""" + + dest_wheel = dest_dir / wheel.name + with InWheel(str(wheel), str(dest_wheel)) as wheel_dir_path: + print(f"repairing {wheel.name} for Windows (DLLs next to wrapper)") + wheel_dir = Path(wheel_dir_path) + for dll in wheel_dir.joinpath("libzim").rglob("*.dll"): + print(f"> moving {dll} using {dll.relative_to(wheel_dir).parent}") + dll.replace(wheel_dir / dll.name) + @property def header_file(self) -> pathlib.Path: return self.base_dir / "include" / "zim" / "zim.h" @@ -381,14 +407,19 @@ def get_cython_extension(): print("Using local libzim binary. Set `USE_SYSTEM_LIBZIM` otherwise.") include_dirs.append("include") library_dirs = ["libzim"] - runtime_library_dirs = ( - [f"@loader_path/libzim/{config.libzim_fname}"] - if sysplatform == "Darwin" - else ["$ORIGIN/libzim/"] - ) + + if config.platform != "Windows": + runtime_library_dirs = ( + [f"@loader_path/libzim/{config.libzim_fname}"] + if sysplatform == "Darwin" + else ["$ORIGIN/libzim/"] + ) extra_compile_args = ["-std=c++11", "-Wall"] - if config.platform != "Windows": + if config.platform == "Windows": + extra_compile_args.append("/MDd" if config.use_msvc_debug else "/MD") + ... + else: extra_compile_args.append("-Wextra") wrapper_extension = Extension( @@ -542,7 +573,29 @@ def run(self): config.cleanup() -if len(sys.argv) == 2 and sys.argv[1] in config.buildless_commands: +class RepairWindowsWheel(Command): + user_options = [ # noqa: RUF012 + ("wheel=", None, "Wheel to repair"), + ("destdir=", None, "Destination folder for repaired wheels"), + ] + + def initialize_options(self): + self.wheel = None + self.destdir = None + + def finalize_options(self): + assert Path(self.wheel).exists(), "wheel file does not exists" + assert ( + Path(self.destdir).exists() and Path(self.destdir).is_dir() + ), "dest_dir does not exists" + + def run(self): + config.repair_windows_wheel(wheel=Path(self.wheel), dest_dir=Path(self.destdir)) + + +if len(sys.argv) == 1 or ( + len(sys.argv) == 2 and sys.argv[1] in config.buildless_commands +): ext_modules = None else: ext_modules = get_cython_extension() @@ -552,6 +605,7 @@ def run(self): "build_ext": LibzimBuildExt, "download_libzim": DownloadLibzim, "clean": LibzimClean, + "repair_win_wheel": RepairWindowsWheel, }, ext_modules=ext_modules, ) diff --git a/tests/test_libzim_creator.py b/tests/test_libzim_creator.py index f39dacb..bdcb84a 100644 --- a/tests/test_libzim_creator.py +++ b/tests/test_libzim_creator.py @@ -168,7 +168,7 @@ def get_creator_output(fpath, verbose): return ps.stdout -@pytest.mark.parametrize("verbose", [(True, False)]) +@pytest.mark.parametrize("verbose", [True, False]) def test_creator_verbose(fpath, verbose): output = get_creator_output(fpath, verbose).strip() lines = output.splitlines() From 963c407ecf2e9c60d104fd3497c65f12b12979d1 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 14:47:49 +0000 Subject: [PATCH 08/33] marking path a raw path to prevent unicodedecode error on windows (C:\Users) --- tests/test_libzim_creator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_libzim_creator.py b/tests/test_libzim_creator.py index bdcb84a..d34c2b8 100644 --- a/tests/test_libzim_creator.py +++ b/tests/test_libzim_creator.py @@ -151,7 +151,7 @@ def get_creator_output(fpath, verbose): """run creator with configVerbose(verbose) and return its stdout as str""" code = """ from libzim.writer import Creator -with Creator("{fpath}").config_verbose({verbose}) as creator: +with Creator(r"{fpath}").config_verbose({verbose}) as creator: pass """.replace( "{fpath}", str(fpath) @@ -162,7 +162,7 @@ def get_creator_output(fpath, verbose): [sys.executable, "-c", code], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - universal_newlines=True, + text=True, ) assert ps.returncode == 0 return ps.stdout From 5ecbc5c85df6c88d99d8d6f8e96769ef63bc6d3d Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 14:54:57 +0000 Subject: [PATCH 09/33] delete Archive so there's no concurrent access to same path --- tests/test_libzim_creator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_libzim_creator.py b/tests/test_libzim_creator.py index d34c2b8..89c2a63 100644 --- a/tests/test_libzim_creator.py +++ b/tests/test_libzim_creator.py @@ -323,6 +323,7 @@ def test_creator_mainpath(fpath, lipsum_item): assert zim.has_main_entry is True assert zim.main_entry.path == "mainPage" assert zim.main_entry.get_item().path == main_path + del zim fpath.unlink() From 6048957652652c78dc0f0bf8c945a4727b936742 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 15:01:00 +0000 Subject: [PATCH 10/33] skip root-only perm bad filename check on Windows --- tests/test_libzim_creator.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_libzim_creator.py b/tests/test_libzim_creator.py index 89c2a63..046f978 100644 --- a/tests/test_libzim_creator.py +++ b/tests/test_libzim_creator.py @@ -1,17 +1,17 @@ #!/usr/bin/env python +from __future__ import annotations import base64 import datetime import itertools import os import pathlib +import platform import subprocess import sys -from typing import Dict - -import pytest import libzim.writer +import pytest from libzim.reader import Archive from libzim.search import Query, Searcher from libzim.suggestion import SuggestionSearcher @@ -49,7 +49,7 @@ def get_contentprovider(self) -> libzim.writer.ContentProvider: return FileProvider(filepath=self.filepath) return StringProvider(content=getattr(self, "content", "")) - def get_hints(self) -> Dict[Hint, int]: + def get_hints(self) -> dict[Hint, int]: return getattr(self, "hints", {Hint.FRONT_ARTICLE: True}) @@ -805,7 +805,7 @@ def get_contentprovider(self): def test_creator_badfilename(tmpdir): - if os.getuid() != 0: + if platform.system() != "Windows" and os.getuid() != 0: # lack of perm with pytest.raises(IOError): Creator("/root/test.zim") From 1064dc85a46704293e2a5e1ec13e8512bb14693c Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 15:18:21 +0000 Subject: [PATCH 11/33] linting update --- setup.py | 66 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/setup.py b/setup.py index af6268a..a870446 100755 --- a/setup.py +++ b/setup.py @@ -32,17 +32,19 @@ class Config: libzim_dl_version: str = os.getenv("LIBZIM_DL_VERSION", "9.1.0") - use_system_libzim: bool = bool(os.getenv("USE_SYSTEM_LIBZIM", False)) - download_libzim: bool = not bool(os.getenv("DONT_DOWNLOAD_LIBZIM", False)) + use_system_libzim: bool = bool(os.getenv("USE_SYSTEM_LIBZIM") or False) + download_libzim: bool = not bool(os.getenv("DONT_DOWNLOAD_LIBZIM") or False) # toggle profiling for coverage report in Cython profiling: bool = os.getenv("PROFILE", "") == "1" # macOS signing - should_sign_apple: bool = bool(os.getenv("SIGN_APPLE", False)) - apple_signing_identity: str = os.getenv("APPLE_SIGNING_IDENTITY") - apple_signing_keychain: str = os.getenv("APPLE_SIGNING_KEYCHAIN_PATH") - apple_signing_keychain_profile: str = os.getenv("APPLE_SIGNING_KEYCHAIN_PROFILE") + should_sign_apple: bool = bool(os.getenv("SIGN_APPLE") or False) + apple_signing_identity: str = os.getenv("APPLE_SIGNING_IDENTITY") or "" + apple_signing_keychain: str = os.getenv("APPLE_SIGNING_KEYCHAIN_PATH") or "" + apple_signing_keychain_profile: str = ( + os.getenv("APPLE_SIGNING_KEYCHAIN_PROFILE") or "" + ) # windows _msvc_debug: bool = bool(os.getenv("MSVC_DEBUG")) @@ -69,11 +71,11 @@ class Config: @property def libzim_major(self) -> str: # assuming nightlies are for version 8.x - return 9 if self.is_nightly else self.libzim_dl_version[0] + return "9" if self.is_nightly else self.libzim_dl_version[0] @property def found_libzim(self) -> str: - return find_library("zim") + return find_library("zim") or "" @property def is_latest_nightly(self) -> bool: @@ -82,8 +84,8 @@ def is_latest_nightly(self) -> bool: @property def is_nightly(self) -> bool: - return self.is_latest_nightly or re.match( - r"\d{4}-\d{2}-\d{2}", self.libzim_dl_version + return self.is_latest_nightly or bool( + re.match(r"\d{4}-\d{2}-\d{2}", self.libzim_dl_version) ) @property @@ -145,7 +147,10 @@ def archive_format(self): def is_musl(self) -> bool: """whether running on a musl system (Alpine)""" ps = subprocess.run( - ["/usr/bin/env", "ldd", "--version"], capture_output=True, text=True + ["/usr/bin/env", "ldd", "--version"], + capture_output=True, + text=True, + check=False, ) try: return "musl libc" in ps.stderr.splitlines()[0] @@ -250,7 +255,7 @@ def _download_and_extract(self, filename: str) -> pathlib.Path: # download a local copy if none present if not fpath.exists(): print(f"> from {url}") - with urllib.request.urlopen(url) as response, open( # nosec + with urllib.request.urlopen(url) as response, open( # nosec # noqa: S310 fpath, "wb" ) as fh: # nosec fh.write(response.read()) @@ -264,7 +269,7 @@ def _download_and_extract(self, filename: str) -> pathlib.Path: # TODO: FIX for zip if self.is_latest_nightly: tar = tarfile.open(fpath) - folder = pathlib.Path(pathlib.Path(tar.firstmember.name).parts[0]) + folder = pathlib.Path(pathlib.Path(tar.getmembers()[0].name).parts[0]) else: folder = fpath.with_name(fpath.name.replace(self.archive_suffix, "")) # unless for ZIP, extract to current folder (all files inside an in-tar folder) @@ -300,7 +305,7 @@ def _install_from(self, folder: pathlib.Path): # remove temp folder shutil.rmtree(folder, ignore_errors=True) - assert self.base_dir.joinpath("include", "zim", "zim.h").exists() + assert self.base_dir.joinpath("include", "zim", "zim.h").exists() # noqa: S101 if config.platform == "Darwin": print("> ensure libzim is notarized") @@ -370,13 +375,13 @@ def can_sign_apple(self) -> bool: config = Config() -def get_cython_extension(): +def get_cython_extension() -> Extension: define_macros = [] compiler_directives = {"language_level": "3"} if config.profiling: define_macros += [("CYTHON_TRACE", "1"), ("CYTHON_TRACE_NOGIL", "1")] - compiler_directives.update(linetrace=True) + compiler_directives.update(linetrace="true") include_dirs = [] library_dirs = [] @@ -384,7 +389,7 @@ def get_cython_extension(): if config.use_system_libzim: if not config.found_libzim: - raise EnvironmentError( + raise OSError( "[!] The libzim library cannot be found.\n" "Please verify it is correctly installed and can be found." ) @@ -399,7 +404,7 @@ def get_cython_extension(): # Check for the CPP Libzim library headers in expected directory if not config.header_file.exists() or not config.dylib_file.exists(): - raise EnvironmentError( + raise OSError( "Unable to find a local copy of libzim " f"at {config.header_file} and {config.dylib_file}" ) @@ -480,7 +485,7 @@ def sign_extension_macos(self, ext): print("Signing & Notarization of the extension") if not config.can_sign_apple: - raise EnvironmentError("Can't sign for apple. Missing information") + raise OSError("Can't sign for apple. Missing information") ext_fpath = pathlib.Path(self.get_ext_fullpath(ext.name)) @@ -510,7 +515,8 @@ def sign_extension_macos(self, ext): "--keepParent", str(ext_fpath), str(ext_zip), - ] + ], + check=True, ) print("> request notarization") @@ -552,7 +558,7 @@ def sign_extension_macos(self, ext): class DownloadLibzim(Command): """dedicated command to solely download libzim binary""" - user_options = [] + user_options = [] # noqa: RUF012 def initialize_options(self): ... @@ -563,7 +569,7 @@ def run(self): class LibzimClean(Command): - user_options = [] + user_options = [] # noqa: RUF012 def initialize_options(self): ... @@ -580,12 +586,14 @@ class RepairWindowsWheel(Command): ] def initialize_options(self): - self.wheel = None - self.destdir = None + self.wheel: str = "" + self.destdir: str = "" def finalize_options(self): - assert Path(self.wheel).exists(), "wheel file does not exists" - assert ( + assert ( # noqa: S101 + self.wheel and Path(self.wheel).exists() + ), "wheel file does not exists" + assert self.destdir and ( # noqa: S101 Path(self.destdir).exists() and Path(self.destdir).is_dir() ), "dest_dir does not exists" @@ -594,11 +602,11 @@ def run(self): if len(sys.argv) == 1 or ( - len(sys.argv) == 2 and sys.argv[1] in config.buildless_commands + len(sys.argv) == 2 and sys.argv[1] in config.buildless_commands # noqa: PLR2004 ): - ext_modules = None + ext_modules = [] else: - ext_modules = get_cython_extension() + ext_modules = [get_cython_extension()] setup( cmdclass={ From 5ed2ae5791358ff4761b1fba34e3e9aaca04a748 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 15:41:21 +0000 Subject: [PATCH 12/33] added missing lastest-nightly folder name --- setup.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index a870446..34247e8 100755 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ import sysconfig import tarfile import urllib.request +import zipfile from ctypes.util import find_library from pathlib import Path @@ -266,12 +267,15 @@ def _download_and_extract(self, filename: str) -> pathlib.Path: # nightly have different download name and extracted folder name as it # uses a redirect - # TODO: FIX for zip if self.is_latest_nightly: - tar = tarfile.open(fpath) - folder = pathlib.Path(pathlib.Path(tar.getmembers()[0].name).parts[0]) + if self.archive_format == "zip": + zf = zipfile.ZipFile(fpath) + folder = pathlib.Path(pathlib.Path(zf.namelist()[0]).stem) + else: + tf = tarfile.open(fpath) + folder = pathlib.Path(pathlib.Path(tf.getmembers()[0].name).stem) else: - folder = fpath.with_name(fpath.name.replace(self.archive_suffix, "")) + folder = fpath.with_name(fpath.stem) # unless for ZIP, extract to current folder (all files inside an in-tar folder) extract_to = folder if self.archive_format == "zip" else self.base_dir shutil.unpack_archive(fpath, extract_to, self.archive_format) From 02b7879efd0f9bdc984b010a2ece1d230edea3fe Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 15:44:43 +0000 Subject: [PATCH 13/33] install build tools so there are present for repair command --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 4eb6c52..facf466 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ manylinux-pypy_x86_64-image = "manylinux_2_28" manylinux-pypy_aarch64-image = "manylinux_2_28" [tool.cibuildwheel.windows] +before-build = "pip install setuptools cython delocate" repair-wheel-command = "python.exe setup.py repair_win_wheel --destdir={dest_dir} --wheel={wheel}" [tool.cibuildwheel.linux] From eddce79160fe178873724bae7412f13926354dc7 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 17:35:18 +0000 Subject: [PATCH 14/33] get_cython_extension already returns a list --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 34247e8..c374cfc 100755 --- a/setup.py +++ b/setup.py @@ -610,7 +610,7 @@ def run(self): ): ext_modules = [] else: - ext_modules = [get_cython_extension()] + ext_modules = get_cython_extension() setup( cmdclass={ From ddefd0a3588b1f4c5776bee9d1e4fb124f9082ba Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 17:43:45 +0000 Subject: [PATCH 15/33] bionic files have been renamed manylinux --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c374cfc..a313f8b 100755 --- a/setup.py +++ b/setup.py @@ -187,7 +187,7 @@ def get_download_filename(self, arch: str | None = None) -> str: variant = "" if self.platform == "Linux": - variant = "-musl" if self.is_musl else "-bionic" + variant = "-musl" if self.is_musl else "-manylinux" if self.is_latest_nightly: version_suffix = "" From bf65e9397b284a140d5430eff64457b0fe0fbc2d Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 17:52:44 +0000 Subject: [PATCH 16/33] dont repair non-win wheels --- setup.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index a313f8b..720cbac 100755 --- a/setup.py +++ b/setup.py @@ -270,12 +270,12 @@ def _download_and_extract(self, filename: str) -> pathlib.Path: if self.is_latest_nightly: if self.archive_format == "zip": zf = zipfile.ZipFile(fpath) - folder = pathlib.Path(pathlib.Path(zf.namelist()[0]).stem) + folder = pathlib.Path(pathlib.Path(zf.namelist()[0]).parts[0]) else: tf = tarfile.open(fpath) - folder = pathlib.Path(pathlib.Path(tf.getmembers()[0].name).stem) + folder = pathlib.Path(pathlib.Path(tf.getmembers()[0].name).parts[0]) else: - folder = fpath.with_name(fpath.stem) + folder = fpath.with_name(fpath.name.replace(self.archive_suffix, "")) # unless for ZIP, extract to current folder (all files inside an in-tar folder) extract_to = folder if self.archive_format == "zip" else self.base_dir shutil.unpack_archive(fpath, extract_to, self.archive_format) @@ -347,6 +347,10 @@ def repair_windows_wheel(self, wheel: Path, dest_dir: Path): """opens windows wheels in target folder and moves all DLLs files inside subdirectories of the wheel to the root one (where wrapper is expected)""" + # we're only interested in windows wheels + if not re.match(r"libzim-.+-win_.+", wheel.stem): + return + dest_wheel = dest_dir / wheel.name with InWheel(str(wheel), str(dest_wheel)) as wheel_dir_path: print(f"repairing {wheel.name} for Windows (DLLs next to wrapper)") From c2948a5016ee3a8c23800138b93d7f23f2e89c7a Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 17:56:50 +0000 Subject: [PATCH 17/33] =?UTF-8?q?satisfy=20isort=20=F0=9F=98=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_libzim_creator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_libzim_creator.py b/tests/test_libzim_creator.py index 046f978..ebf652f 100644 --- a/tests/test_libzim_creator.py +++ b/tests/test_libzim_creator.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + from __future__ import annotations import base64 @@ -10,8 +11,9 @@ import subprocess import sys -import libzim.writer import pytest + +import libzim.writer from libzim.reader import Archive from libzim.search import Query, Searcher from libzim.suggestion import SuggestionSearcher From 5b7ccb6212ffeb1a0f2343a6691077d5f610ea1d Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 18:08:44 +0000 Subject: [PATCH 18/33] fix editable install on windows for tests --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9c2f800..f285a0b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,6 +47,10 @@ jobs: PROFILE: 1 run: pip install -e . + - name: move DLLs next to wrapper + if: matrix.os == 'windows-latest' + run: move libzim\*.dll %cd%\ + - name: Testing run: | pip install pytest pytest-cov cython From 89f027ac71c1b797f74d835c7a4a037bc949a8bf Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 18:20:44 +0000 Subject: [PATCH 19/33] satisfy old black? --- setup.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 720cbac..bbc3d53 100755 --- a/setup.py +++ b/setup.py @@ -568,9 +568,11 @@ class DownloadLibzim(Command): user_options = [] # noqa: RUF012 - def initialize_options(self): ... + def initialize_options(self): + ... - def finalize_options(self): ... + def finalize_options(self): + ... def run(self): config.download_to_dest() @@ -579,9 +581,11 @@ def run(self): class LibzimClean(Command): user_options = [] # noqa: RUF012 - def initialize_options(self): ... + def initialize_options(self): + ... - def finalize_options(self): ... + def finalize_options(self): + ... def run(self): config.cleanup() From 4a179fc1080c624c1f5fa174c4520e03ea03bc4e Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 18:21:17 +0000 Subject: [PATCH 20/33] [tmp] ls folder to find out issue --- .github/workflows/test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f285a0b..aaeeda7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,7 +49,9 @@ jobs: - name: move DLLs next to wrapper if: matrix.os == 'windows-latest' - run: move libzim\*.dll %cd%\ + run: | + dir + move libzim\*.dll %cd%\ - name: Testing run: | From 6dd986fc62f00e35519f1c44bfb4d0a0dba2d2fc Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 18:52:38 +0000 Subject: [PATCH 21/33] sdist should not download binary --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index bbc3d53..d0f9dda 100755 --- a/setup.py +++ b/setup.py @@ -67,6 +67,8 @@ class Config: "egg_info", "--version", "download_libzim", + "build_sdist", + "sdist", ) @property @@ -383,7 +385,7 @@ def can_sign_apple(self) -> bool: config = Config() -def get_cython_extension() -> Extension: +def get_cython_extension() -> list[Extension]: define_macros = [] compiler_directives = {"language_level": "3"} @@ -614,7 +616,7 @@ def run(self): if len(sys.argv) == 1 or ( - len(sys.argv) == 2 and sys.argv[1] in config.buildless_commands # noqa: PLR2004 + len(sys.argv) >= 2 and sys.argv[1] in config.buildless_commands # noqa: PLR2004 ): ext_modules = [] else: From 7b3cf4ddda1b88cb465ca793adbb748d25751dab Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 19:03:22 +0000 Subject: [PATCH 22/33] use move-item --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aaeeda7..3a0e5ba 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,7 +51,7 @@ jobs: if: matrix.os == 'windows-latest' run: | dir - move libzim\*.dll %cd%\ + Move-Item -Force -Path .\libzim\*.dll -Destination .\ - name: Testing run: | From 6ee06d5eb0695685305c1e16ec21cdbadbd4257d Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 19:17:07 +0000 Subject: [PATCH 23/33] restoring other platforms and versions --- .github/workflows/release.yaml | 2 +- .github/workflows/test.yml | 6 +++--- .github/workflows/wheels.yml | 2 +- pyproject.toml | 5 ++--- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 908730a..e4f0fd2 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -22,7 +22,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, macos-13] # windows-2019 + os: [ubuntu-20.04, macos-13, windows-2022] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3a0e5ba..6f19e83 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,8 +27,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-latest] - python: ["3.12"] + os: [ubuntu-22.04, macos-13, windows-2022] + python: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 @@ -48,7 +48,7 @@ jobs: run: pip install -e . - name: move DLLs next to wrapper - if: matrix.os == 'windows-latest' + if: matrix.os == 'windows-2022' run: | dir Move-Item -Force -Path .\libzim\*.dll -Destination .\ diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index aec86c5..90767b0 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -20,7 +20,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-latest] + os: [ubuntu-20.04, macos-13, windows-2022] steps: - uses: actions/checkout@v3 diff --git a/pyproject.toml b/pyproject.toml index facf466..1d0628d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,10 +11,9 @@ testpaths = ["tests"] pythonpath = ["."] [tool.cibuildwheel] -build = "cp312-win_amd64" -# disabling windows until hhttps://github.com/kiwix/kiwix-build/issues/466 +build = "*" # disabling PyPy due to 2 failing tests -# skip = "pp* *-win*" +skip = "pp*" test-requires = ["pytest"] test-command = "py.test {project}/tests/" From 11eb8d3d24bc8d650b2d49b2d0d27d3e6269f4d9 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 19:29:13 +0000 Subject: [PATCH 24/33] disabling linux until https://github.com/kiwix/kiwix-build/issues/746 --- .github/workflows/test.yml | 2 +- .github/workflows/wheels.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6f19e83..9449e92 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,7 +27,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04, macos-13, windows-2022] + os: [macos-13, windows-2022] # ubuntu-22.04 python: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 90767b0..2528110 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -20,7 +20,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, macos-13, windows-2022] + os: [macos-13, windows-2022] # ubuntu-20.04 steps: - uses: actions/checkout@v3 From 29fab5e840a482a4f72dac42cc92606ca3f0e1ad Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 19:35:37 +0000 Subject: [PATCH 25/33] disable win32 as we only support 64b --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1d0628d..60aacb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ pythonpath = ["."] [tool.cibuildwheel] build = "*" # disabling PyPy due to 2 failing tests -skip = "pp*" +skip = "pp* *win32*" test-requires = ["pytest"] test-command = "py.test {project}/tests/" From dd41f2d7c1355e25a8b0707bb141d0ba660b1f28 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 19:41:35 +0000 Subject: [PATCH 26/33] updated actions versions --- .github/workflows/release.yaml | 12 ++++++------ .github/workflows/test.yml | 10 +++++----- .github/workflows/wheels.yml | 10 +++++----- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e4f0fd2..03a2a22 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -25,7 +25,7 @@ jobs: os: [ubuntu-20.04, macos-13, windows-2022] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up QEMU if: runner.os == 'Linux' @@ -68,7 +68,7 @@ jobs: security unlock-keychain -p mysecretpassword ${APPLE_SIGNING_KEYCHAIN_PATH} - name: Build wheels - uses: pypa/cibuildwheel@v2.16 + uses: pypa/cibuildwheel@v2.20 - name: Cleanup Apple Keychain if: matrix.os == 'macos-13' @@ -86,12 +86,12 @@ jobs: name: Build source distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build sdist run: pipx run build --sdist - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: path: dist/*.tar.gz @@ -100,14 +100,14 @@ jobs: runs-on: ubuntu-latest environment: release steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: # unpacks default artifact into dist/ # if `name: artifact` is omitted, the action will create extra parent dir name: artifact path: dist - - uses: pypa/gh-action-pypi-publish@v1.8.10 + - uses: pypa/gh-action-pypi-publish@v1.9.0 with: user: __token__ # password: ${{ secrets.PYPI_TEST_API_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9449e92..f45c526 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,10 +9,10 @@ jobs: lint: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.12" architecture: x64 @@ -31,10 +31,10 @@ jobs: python: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} architecture: x64 @@ -60,6 +60,6 @@ jobs: - name: Upload coverage report to codecov if: matrix.os == 'ubuntu-22.04' && matrix.python == '3.11' - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 2528110..187a403 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -23,18 +23,18 @@ jobs: os: [macos-13, windows-2022] # ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up QEMU if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: all - name: Build wheels uses: pypa/cibuildwheel@v2.16 - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: path: ./wheelhouse/*.whl @@ -42,11 +42,11 @@ jobs: name: Build source distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build sdist run: pipx run build --sdist - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: path: dist/*.tar.gz From e0250b40bb475be89855358814f786054bf967b2 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 20:01:40 +0000 Subject: [PATCH 27/33] upload-artifact@v4 requires unique name, reverting for now --- .github/workflows/release.yaml | 2 +- .github/workflows/wheels.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 03a2a22..5406123 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -91,7 +91,7 @@ jobs: - name: Build sdist run: pipx run build --sdist - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v3 with: path: dist/*.tar.gz diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 187a403..25e5c7e 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -34,7 +34,7 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.16 - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v3 with: path: ./wheelhouse/*.whl @@ -47,6 +47,6 @@ jobs: - name: Build sdist run: pipx run build --sdist - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v3 with: path: dist/*.tar.gz From 398748c39c8b05e5bfaac1bd5448656fcbf947cd Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 20:19:35 +0000 Subject: [PATCH 28/33] removed temp dir listing --- .github/workflows/test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f45c526..49c9af1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,9 +49,7 @@ jobs: - name: move DLLs next to wrapper if: matrix.os == 'windows-2022' - run: | - dir - Move-Item -Force -Path .\libzim\*.dll -Destination .\ + run: Move-Item -Force -Path .\libzim\*.dll -Destination .\ - name: Testing run: | From 1f976134a45868937d315d7229e8400f539794d7 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Sat, 31 Aug 2024 20:21:19 +0000 Subject: [PATCH 29/33] updated readme and changelog --- CHANGELOG.md | 6 ++++++ README.md | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b36843..6c18aa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleazed + +### Added + +- Windows (x64) support (#91) + ## [3.4.0] - 2023-12-16 ### Added diff --git a/README.md b/README.md index 0fb27a2..0a4e68b 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,25 @@ with Creator("test.zim") as creator: | `APPLE_SIGNING_KEYCHAIN_PATH` | `/tmp/build.keychain` | Path to the Keychain containing the certificate to sign for macOS with | | `APPLE_SIGNING_KEYCHAIN_PROFILE` | `build` | Name of the profile in the specified Keychain | + +### Building on Windows + +On Windows, built wheels needs to be fixed post-build to move the bundled DLLs (libzim and libicu) +next to the wrapper (Windows does not support runtime path). + +After building you wheel, run + +```ps +python setup.py repair_win_wheel --wheel=dist/xxx.whl --destdir wheels\ +``` + +Similarily, if you install as editable (`pip install -e .`), you need to place those DLLs at the root +of the repo. + +```ps +Move-Item -Force -Path .\libzim\*.dll -Destination .\ +``` + ### Examples ##### Default: downloading and bundling most appropriate libzim release binary From 2dca8adcdbfa4a3f2fceb31a8423c1eaba4b7006 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Mon, 2 Sep 2024 10:57:56 +0000 Subject: [PATCH 30/33] fixed libzim bin in lib64/ and lib// folders --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d0f9dda..5ef0bd8 100755 --- a/setup.py +++ b/setup.py @@ -296,8 +296,8 @@ def _install_from(self, folder: pathlib.Path): (self.base_dir / "include").mkdir(exist_ok=True, parents=True) shutil.move(folder / "include" / "zim", self.base_dir / "include" / "zim") - # copy new libs - for fpath in folder.joinpath("lib").rglob("libzim.*"): + # copy new libs (from lib/, lib/ or lib64/) + for fpath in folder.rglob("lib*/**/libzim.*"): print(f"{fpath} -> {libzim_dir / fpath.name}") os.replace(fpath, libzim_dir / fpath.name) # windows has different folder and name From d5b5b29ebfe6fe25e0652b87201c2e00db23cf0d Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Mon, 2 Sep 2024 16:30:05 +0000 Subject: [PATCH 31/33] using 9.2.3-2 --- .github/workflows/release.yaml | 2 +- .github/workflows/test.yml | 2 +- .github/workflows/wheels.yml | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5406123..b522600 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -6,7 +6,7 @@ on: - published env: - LIBZIM_DL_VERSION: "9.2.3-1" + LIBZIM_DL_VERSION: "9.2.3-2" MACOSX_DEPLOYMENT_TARGET: "12.0" CIBW_ENVIRONMENT_PASS_LINUX: "LIBZIM_DL_VERSION" # APPLE_SIGNING_KEYCHAIN_PATH set in prepare keychain step diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 49c9af1..0aec62f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: test on: [push] env: - LIBZIM_DL_VERSION: "9.2.3-1" + LIBZIM_DL_VERSION: "9.2.3-2" MACOSX_DEPLOYMENT_TARGET: "12.0" jobs: diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 25e5c7e..c2d7b6b 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -8,7 +8,7 @@ on: - windows env: - LIBZIM_DL_VERSION: "9.2.3-1" + LIBZIM_DL_VERSION: "9.2.3-2" MACOSX_DEPLOYMENT_TARGET: "12.0" CIBW_ENVIRONMENT_PASS_LINUX: "LIBZIM_DL_VERSION" CIBW_BUILD_VERBOSITY: "3" diff --git a/setup.py b/setup.py index 5ef0bd8..a3a1451 100755 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ class Config: - libzim_dl_version: str = os.getenv("LIBZIM_DL_VERSION", "9.1.0") + libzim_dl_version: str = os.getenv("LIBZIM_DL_VERSION", "9.2.3-2") use_system_libzim: bool = bool(os.getenv("USE_SYSTEM_LIBZIM") or False) download_libzim: bool = not bool(os.getenv("DONT_DOWNLOAD_LIBZIM") or False) From b40cbbb084d379e9d14d54b7f1db120ea2ac9ac5 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Mon, 2 Sep 2024 16:30:50 +0000 Subject: [PATCH 32/33] enabling linux --- .github/workflows/test.yml | 2 +- .github/workflows/wheels.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0aec62f..d1e9d10 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,7 +27,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-13, windows-2022] # ubuntu-22.04 + os: [macos-13, windows-2022, ubuntu-22.04] python: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index c2d7b6b..b8d4e21 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -20,7 +20,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-13, windows-2022] # ubuntu-20.04 + os: [macos-13, windows-2022, ubuntu-20.04] steps: - uses: actions/checkout@v4 From 9dea8875dff7168831c65ebae82bbf0fb3803000 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Tue, 3 Sep 2024 14:25:12 +0000 Subject: [PATCH 33/33] using upload-artifactv4 --- .github/workflows/release.yaml | 23 ++++++++++++++++++----- .github/workflows/wheels.yml | 6 ++++-- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b522600..8cc2a10 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -78,8 +78,9 @@ jobs: security delete-keychain ${APPLE_SIGNING_KEYCHAIN_PATH} rm -f ${APPLE_SIGNING_KEYCHAIN_PATH} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: + name: wheels-${{ matrix.os }} path: ./wheelhouse/*.whl build_sdist: @@ -91,8 +92,9 @@ jobs: - name: Build sdist run: pipx run build --sdist - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: + name: sdist path: dist/*.tar.gz upload_pypi: @@ -100,11 +102,22 @@ jobs: runs-on: ubuntu-latest environment: release steps: + # retrieve all artifacts - uses: actions/download-artifact@v4 with: - # unpacks default artifact into dist/ - # if `name: artifact` is omitted, the action will create extra parent dir - name: artifact + name: sdist + path: dist + - uses: actions/download-artifact@v4 + with: + name: wheels-ubuntu-20.04 + path: dist + - uses: actions/download-artifact@v4 + with: + name: wheels-macos-13 + path: dist + - uses: actions/download-artifact@v4 + with: + name: wheels-windows-2022 path: dist - uses: pypa/gh-action-pypi-publish@v1.9.0 diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index b8d4e21..1676cf9 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -34,8 +34,9 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.16 - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: + name: wheels-${{ matrix.os }} path: ./wheelhouse/*.whl build_sdist: @@ -47,6 +48,7 @@ jobs: - name: Build sdist run: pipx run build --sdist - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: + name: sdist path: dist/*.tar.gz