refactor: group worker repos under the workers subdirectory

.dockerignore

@@ -1,5 +1,3 @@
-old_temporal_manager
-.data
 .github
 .venv
 docs

.github/workflows/publish_asr_worker.yml

@@ -0,0 +1,108 @@
+name: Publish asr-worker
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - 'asr-worker-*'
+
+jobs:
+  publish-io-worker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: icij/asr-transcription-io-worker
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          # You'll need to set these secrets
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push image
+        uses: docker/build-push-action@v7
+        with:
+          target: io-worker
+          context: workers/asr-worker
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  publish-preprocessing-worker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: icij/asr-transcription-preprocessing-worker
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          # You'll need to set these secrets
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push image
+        uses: docker/build-push-action@v7
+        with:
+          target: preprocessing-worker
+          context: workers/asr-worker
+          platforms: linux/amd64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  publish-inference-gpu-worker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: icij/asr-transcription-inference-gpu-worker
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          # You'll need to set these secrets
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push image
+        uses: docker/build-push-action@v7
+        with:
+          target: inference-gpu-worker
+          context: workers/asr-worker
+          platforms: linux/amd64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false

.github/workflows/test_asr_worker.yml

@@ -6,7 +6,7 @@ on:
   pull_request:
     paths:
       - 'datashare-python'
-      - 'asr-worker'
+      - 'workers/asr-worker'
 
 # TODO: leverage some caching here
 jobs:
@@ -18,9 +18,9 @@ jobs:
         with:
           args: "--version" # skips test by displaying the version
       - name: Check formatting
-        run: ruff format --config qa/ruff.toml --check asr-worker
+        run: ruff format --config qa/ruff.toml --check workers/asr-worker
       - name: Lint test
-        run: ruff check --config qa/ruff.toml asr-worker
+        run: ruff check --config qa/ruff.toml workers/asr-worker
 
   test:
     runs-on: ubuntu-latest
@@ -42,14 +42,14 @@ jobs:
           version: ${{ env.ASTRAL_VERSION }}
           python-version: ${{ env.PYTHON_VERSION }}
           enable-cache: true
-          working-directory: asr-worker
+          working-directory: workers/asr-worker
       - name: Install ffmpeg
         run: |
           sudo apt-get update
           sudo apt-get install -y ffmpeg libavcodec-dev libavformat-dev libavutil-dev
       - name: Run tests
         run: |
-          cd asr-worker
+          cd workers/asr-worker
           uv sync --frozen --all-extras
           uv run --frozen python -m pytest --timeout=180 -vvv --cache-clear --show-capture=all -r A
 

.github/workflows/test_translation_worker.yml

@@ -5,7 +5,7 @@ on:
     branches: [ 'main' ]
   pull_request:
     paths:
-      - 'translation-worker'
+      - 'workers/translation-worker'
 
 # TODO: leverage some caching here
 jobs:
@@ -17,9 +17,9 @@ jobs:
         with:
           args: "--version" # skips test by displaying the version
       - name: Check formatting
-        run: ruff format --config qa/ruff.toml --check translation-worker
+        run: ruff format --config qa/ruff.toml --check workers/translation-worker
       - name: Lint test
-        run: ruff check --config qa/ruff.toml translation-worker
+        run: ruff check --config qa/ruff.toml workers/translation-worker
 
   test:
     runs-on: ubuntu-latest
@@ -41,10 +41,10 @@ jobs:
           version: ${{ env.ASTRAL_VERSION }}
           python-version: ${{ env.PYTHON_VERSION }}
           enable-cache: true
-          working-directory: translation-worker
+          working-directory: workers/translation-worker
       - name: Run tests
         run: |
-          cd translation-worker
+          cd workers/translation-worker
           uv sync --frozen --all-extras
           uv run --frozen python -m pytest --timeout=180 -vvv --cache-clear --show-capture=all -r A
 

Makefile

@@ -1,3 +1,6 @@
+lock-dist:
+	./scripts/lock-dist.sh ${project}
+
 create-venv:
 	[ -d .venv ] || uv venv --python 3.13
 

datashare-python/datashare_python/discovery.py

@@ -58,7 +58,7 @@ def discover(
         if wf_names:
             msg += "workflow patterns " + ", ".join(wf_names) + " "
         if act_names:
-            msg = "activity patterns " + ", ".join(act_names)
+            msg += "activity patterns " + ", ".join(act_names)
         raise ValueError(msg)
     deps = None
     if deps_name is not None:

datashare-python/datashare_python/template.py

@@ -61,7 +61,6 @@ def init_project(name: str, path: Path) -> None:
     pyproject_toml = tomlkit.loads(pyproject_toml_path.read_text())
     pyproject_toml = _update_pyproject_toml(pyproject_toml, package_name=package_name)
     pyproject_toml_path.write_text(tomlkit.dumps(pyproject_toml))
-    raise NotImplementedError("implement entry point and dockerfile init")
 
 
 _BASE_DEPS = {"datashare-python", "icij-common", "temporalio"}

datashare-python/tests/cli/test_worker.py

@@ -38,6 +38,8 @@ async def test_start_workers(
                 "create-translation-batches",
                 "--workflows",
                 "ping",
+                "--dependencies",
+                "base",
                 "--temporal-address",
                 "localhost:7233",
             ],

datashare-python/tests/test_discovery.py

@@ -54,9 +54,9 @@ def test_discover_activities(names: list[str], expected_activities: set[str]) ->
     assert activities == expected_activities
 
 
-@pytest.mark.parametrize("name", ["base", None])
-def test_discover_dependencies(name: str | None) -> None:
+def test_discover_dependencies() -> None:
     # When
+    name = "base"
     deps = discover_dependencies(name)
     # Then
     expected_deps = ["set_loggers", "set_es_client"]
@@ -88,25 +88,4 @@ def mocked_entry_points(name: str, group: str) -> EntryPoints:  # noqa: ARG001
     # When/Then
     expected = "found multiple dependencies for name"
     with pytest.raises(ValueError, match=re.escape(expected)):
-        discover_dependencies(name=None)
-
-
-def test_discover_dependencies_should_raise_for_multiple_entry_points(
-    monkeypatch: MonkeyPatch,
-) -> None:
-    # Given
-    def mocked_entry_points(name: str, group: str) -> EntryPoints:  # noqa: ARG001
-        ep = MagicMock()
-        ep.load.return_value = {"a": [], "b": []}
-        entry_points = MagicMock()
-        entry_points.__getitem__.return_value = ep
-        return entry_points
-
-    monkeypatch.setattr(datashare_python.discovery, "entry_points", mocked_entry_points)
-    # When/Then
-    expected = (
-        'dependency registry contains multiples entries "a", "b",'
-        " please select one by providing a name"
-    )
-    with pytest.raises(ValueError, match=re.escape(expected)):
-        discover_dependencies(name=None)
+        discover_dependencies(name="some_name")

scripts/lock-dist.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+cd "$1"
+# uv bug -n flag to discard cache takes 2 times to work
+uv lock -n --no-sources-package datashare-python || uv lock -n --no-sources-package datashare-python
+cp uv.lock uv.dist.lock
+uv lock