Skip to content

Commit aecd0a3

Browse files
benoit74benoit74
committed
Minimal scraper working from end-to-end
1 parent 2c4a12a commit aecd0a3

80 files changed

Lines changed: 9860 additions & 2 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/FUNDING.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# These are supported funding model platforms
2+
3+
github: kiwix # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
4+
patreon: # Replace with a single Patreon username
5+
open_collective: # Replace with a single Open Collective username
6+
ko_fi: # Replace with a single Ko-fi username
7+
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
8+
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
9+
liberapay: # Replace with a single Liberapay username
10+
issuehunt: # Replace with a single IssueHunt username
11+
otechie: # Replace with a single Otechie username
12+
custom: # https://kiwix.org/support-us/

‎.github/stale.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
daysUntilClose: false
2+
staleLabel: stale
3+
4+
issues:
5+
daysUntilStale: 60
6+
markComment: >
7+
This issue has been automatically marked as stale because it has not had
8+
recent activity. It will be now be reviewed manually. Thank you
9+
for your contributions.
10+
pulls:
11+
daysUntilStale: 7
12+
markComment: >
13+
This pull request has been automatically marked as stale because it has not had
14+
recent activity. It will be now be reviewed manually. Thank you
15+
for your contributions.

‎.github/workflows/Publish.yml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
name: Build and upload to PyPI
2+
3+
on:
4+
release:
5+
types: [published]
6+
7+
jobs:
8+
publish:
9+
runs-on: ubuntu-24.04
10+
permissions:
11+
id-token: write # mandatory for PyPI trusted publishing
12+
13+
steps:
14+
- uses: actions/checkout@v4
15+
16+
- name: Set up Python
17+
uses: actions/setup-python@v5
18+
with:
19+
python-version-file: scraper/pyproject.toml
20+
architecture: x64
21+
22+
- name: Build packages
23+
working-directory: scraper
24+
run: |
25+
pip install -U pip build
26+
python -m build --sdist --wheel
27+
28+
- name: Upload to PyPI
29+
uses: pypa/gh-action-pypi-publish@release/v1.10
30+
with:
31+
packages-dir: scraper/dist/
32+
33+
- name: Build and push Docker image
34+
uses: openzim/docker-publish-action@v10
35+
with:
36+
image-name: openzim/libretexts
37+
tag-pattern: /^v([0-9.]+)$/
38+
latest-on-tag: true
39+
restrict-to: openzim/libretexts
40+
registries: ghcr.io
41+
credentials:
42+
GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
43+
GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
44+
repo_description: auto
45+
repo_overview: auto

‎.github/workflows/PublishDockerDevImage.yaml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: Publish Docker dev image
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
8+
jobs:
9+
publish:
10+
runs-on: ubuntu-24.04
11+
12+
steps:
13+
- uses: actions/checkout@v4
14+
15+
- name: Build and push Docker image
16+
uses: openzim/docker-publish-action@v10
17+
with:
18+
image-name: openzim/libretexts
19+
manual-tag: dev
20+
latest-on-tag: false
21+
restrict-to: openzim/libretexts
22+
registries: ghcr.io
23+
credentials:
24+
GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
25+
GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
26+
repo_description: auto
27+
repo_overview: auto

‎.github/workflows/QA.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
name: QA
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- main
8+
9+
jobs:
10+
check-scraper-qa:
11+
runs-on: ubuntu-24.04
12+
13+
steps:
14+
- uses: actions/checkout@v4
15+
16+
- name: Set up Python
17+
uses: actions/setup-python@v5
18+
with:
19+
python-version-file: scraper/pyproject.toml
20+
architecture: x64
21+
22+
- name: Install dependencies
23+
working-directory: scraper
24+
run: |
25+
pip install -U pip
26+
pip install -e .[lint,check,scripts,test]
27+
28+
- name: Check black formatting
29+
working-directory: scraper
30+
run: inv lint-black
31+
32+
- name: Check ruff
33+
working-directory: scraper
34+
run: inv lint-ruff
35+
36+
- name: Check pyright
37+
working-directory: scraper
38+
run: inv check-pyright
39+
40+
check-zimui-qa:
41+
runs-on: ubuntu-24.04
42+
43+
steps:
44+
- uses: actions/checkout@v4
45+
46+
- name: Set up Node.js
47+
uses: actions/setup-node@v4
48+
with:
49+
node-version-file: zimui/.node-version
50+
51+
- name: Install JS dependencies
52+
working-directory: zimui
53+
run: |
54+
yarn install
55+
- name: Check prettier
56+
working-directory: zimui
57+
run: |
58+
yarn format
59+
- name: Check eslint
60+
working-directory: zimui
61+
run: |
62+
yarn lint

‎.github/workflows/Tests.yml‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
name: Tests
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- main
8+
9+
jobs:
10+
test-scraper:
11+
runs-on: ubuntu-24.04
12+
13+
steps:
14+
- uses: actions/checkout@v4
15+
16+
- name: Set up Python
17+
uses: actions/setup-python@v5
18+
with:
19+
python-version-file: scraper/pyproject.toml
20+
architecture: x64
21+
22+
- name: Install dependencies (and project)
23+
working-directory: scraper
24+
run: |
25+
pip install -U pip
26+
pip install -e .[test,scripts]
27+
28+
- name: Run the tests
29+
working-directory: scraper
30+
run: inv coverage --args "-vvv"
31+
32+
- name: Upload coverage report to codecov
33+
uses: codecov/codecov-action@v4
34+
with:
35+
fail_ci_if_error: true
36+
token: ${{ secrets.CODECOV_TOKEN }}
37+
38+
build-scraper:
39+
runs-on: ubuntu-24.04
40+
steps:
41+
- uses: actions/checkout@v4
42+
43+
- name: Set up Python
44+
uses: actions/setup-python@v5
45+
with:
46+
python-version-file: scraper/pyproject.toml
47+
architecture: x64
48+
49+
- name: Ensure we can build Python targets
50+
working-directory: scraper
51+
run: |
52+
pip install -U pip build
53+
python3 -m build --sdist --wheel
54+
55+
build-and-test-zimui:
56+
runs-on: ubuntu-24.04
57+
steps:
58+
- uses: actions/checkout@v4
59+
60+
- name: Set up Node.js
61+
uses: actions/setup-node@v4
62+
with:
63+
node-version-file: zimui/.node-version
64+
65+
- name: Install dependencies
66+
working-directory: zimui
67+
run: |
68+
yarn install
69+
70+
- name: Test
71+
working-directory: zimui
72+
run: |
73+
yarn test:unit:run
74+
75+
- name: Build
76+
working-directory: zimui
77+
run: |
78+
yarn build
79+
80+
- name: Start web server
81+
working-directory: zimui
82+
run: |
83+
yarn preview &
84+
85+
- name: Wait for web server to be ready
86+
run: |
87+
npx wait-on http://localhost:4173
88+
89+
- name: Run frontend tests
90+
working-directory: zimui
91+
run: |
92+
$(yarn bin)/cypress run
93+
94+
# this job replaces the standard "build_docker" job since it builds the docker image
95+
run-integration-tests:
96+
runs-on: ubuntu-24.04
97+
steps:
98+
- uses: actions/checkout@v4
99+
100+
- name: Ensure we can build the Docker image
101+
run: |
102+
docker build -t libretexts2zim .
103+
104+
- name: Run scraper
105+
run: docker run -v $PWD/output:/output libretexts2zim libretexts2zim --library-slug geo --library-name Geosciences --file-name-format "tests_en_libretexts-geo"
106+
107+
- name: Run integration test suite
108+
run: docker run -v $PWD/scraper/tests-integration:/src/scraper/tests-integration -v $PWD/output:/output libretexts2zim bash -c "pip install pytest; pytest -v /src/scraper/tests-integration"

0 commit comments

Comments
 (0)