feat: implement the FileAutomation 1.0 roadmap

.gitattributes

@@ -1,2 +1,4 @@
 # Auto detect text files and perform LF normalization
 * text=auto
+# Shell scripts run under bash on Linux runners, which does not accept CRLF.
+*.sh text eol=lf

.github/requirements/integration.in

@@ -0,0 +1,33 @@
+# Python 3.12 integration dependencies, locked for all runner platforms.
+# Regenerate: uv pip compile --universal --python-version 3.12 --generate-hashes
+# --only-binary :all: --exclude-newer 2026-10-02T00:00:00Z
+# .github/requirements/integration.in -o .github/requirements/integration.txt
+setuptools>=77
+wheel
+requests>=2.31.0
+tqdm>=4.66.0
+watchdog>=4.0.0
+cryptography>=50.0.0
+prometheus_client>=0.26.0
+defusedxml>=0.7.1
+je_action_core>=0.0.2
+PyYAML>=6.0.3
+tzdata>=2024.1; platform_system == 'Windows'
+opentelemetry-api>=1.44.0
+opentelemetry-sdk>=1.44.0
+tomli>=2.0.1; python_version<"3.11"
+boto3>=1.34.0
+azure-storage-blob>=12.19.0
+google-api-python-client>=2.100.0
+google-auth-httplib2>=0.2.0
+google-auth-oauthlib>=1.2.0
+dropbox>=11.36.2
+paramiko>=3.4.0
+smbprotocol>=1.13.0
+fsspec>=2024.2.0
+msal>=1.39.0
+boxsdk>=10.15.0,<11
+pyarrow>=25.0.1
+PySide6>=6.11.2,<6.12
+pytest>=8.0.0
+pytest-cov>=5.0.0

.github/requirements/lint.in

@@ -0,0 +1,6 @@
+# Regenerate with uv pip compile --generate-hashes --python-version 3.12
+# --only-binary :all: --exclude-newer 2026-10-02T00:00:00Z
+# .github/requirements/lint.in -o .github/requirements/lint.txt
+ruff
+mypy
+je_action_core>=0.0.2

.github/workflows/ci-dev.yml

@@ -25,9 +25,8 @@ jobs:
           cache: pip
       - name: Install tooling
         run: |
-          python -m pip install --upgrade pip
-          # mypy reads je_action_core's types (py.typed) for the classes built on it
-          pip install ruff mypy "je_action_core>=0.0.2"
+          # mypy reads the action engine types from the hash-locked tooling.
+          python -m pip install --require-hashes --only-binary :all: -r .github/requirements/lint.txt
       - name: Ruff check
         run: ruff check automation_file tests
       - name: Ruff format check
@@ -56,8 +55,8 @@ jobs:
         run: |
           python -m pip install --upgrade pip wheel
           Copy-Item dev.toml pyproject.toml -Force
-          pip install -e .
-          pip install pytest pytest-cov
+          # Every extra, so every backend's tests run. The minimal job runs without any.
+          pip install -e ".[all,test]"
       - name: Run pytest with coverage
         run: python -m pytest tests/ -v --tb=short --cov=automation_file --cov-report=term-missing --cov-report=xml
       - name: Upload coverage artifact
@@ -67,12 +66,84 @@ jobs:
           name: coverage-xml
           path: coverage.xml
 
+  minimal:
+    # The base install: no cloud SDK and no GUI toolkit. The tests of a backend whose extra is
+    # missing skip; everything else, the import guard included, has to pass.
+    needs: lint
+    runs-on: windows-latest
+    timeout-minutes: 15  # no run yet: the floor of 15 minutes, to revisit after the first runs
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install the base package
+        run: |
+          python -m pip install --upgrade pip wheel
+          Copy-Item dev.toml pyproject.toml -Force
+          pip install -e ".[test]"
+      - name: Run pytest
+        run: python -m pytest tests/ -v --tb=short
+
+  extras:
+    # Each extra on its own: it installs, and its backend's tests run with no other SDK present.
+    needs: lint
+    runs-on: windows-latest
+    timeout-minutes: 15  # no run yet: the floor of 15 minutes, to revisit after the first runs
+    strategy:
+      fail-fast: false
+      matrix:
+        extra: [ s3, azure, gdrive, dropbox, sftp, ftp, webdav, smb, fsspec, onedrive, box, parquet, gui ]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install the package with one extra
+        run: |
+          python -m pip install --upgrade pip wheel
+          Copy-Item dev.toml pyproject.toml -Force
+          pip install -e ".[${{ matrix.extra }},test]"
+      - name: Run pytest
+        run: python -m pytest tests/ -v --tb=short
+
+  package:
+    # The sdist and the wheel build from dev.toml and their metadata renders, before anything is
+    # merged. It gates nothing: the publish job builds again from the locked tools.
+    needs: lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 15  # no run yet: the floor of 15 minutes, to revisit after the first runs
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Build the distribution and check its metadata
+        run: |
+          # The locked, wheels-only tools the publish jobs build with.
+          python -m pip install --require-hashes --only-binary :all: -r .github/requirements/publish.txt
+          cp dev.toml pyproject.toml
+          python -m build --no-isolation
+          twine check dist/*
+
   publish-dev:
     # The dev channel. A push to dev that passes lint and the tests is built from dev.toml and uploaded
     # when it is still the tip of dev and ships something the newest automation_file_dev does not.
     # scripts/dev_release.py picks the version from PyPI, so nothing is committed back.
     name: Publish automation_file_dev to PyPI
-    needs: [lint, pytest]
+    needs: [lint, pytest, minimal, extras]
     if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
     runs-on: ubuntu-latest
     timeout-minutes: 15  # about 3x the slowest recent run, at least 15

.github/workflows/ci-stable.yml

@@ -25,9 +25,8 @@ jobs:
           cache: pip
       - name: Install tooling
         run: |
-          python -m pip install --upgrade pip
-          # mypy reads je_action_core's types (py.typed) for the classes built on it
-          pip install ruff mypy "je_action_core>=0.0.2"
+          # mypy reads the action engine types from the hash-locked tooling.
+          python -m pip install --require-hashes --only-binary :all: -r .github/requirements/lint.txt
       - name: Ruff check
         run: ruff check automation_file tests
       - name: Ruff format check
@@ -56,8 +55,8 @@ jobs:
         run: |
           python -m pip install --upgrade pip wheel
           Copy-Item stable.toml pyproject.toml -Force
-          pip install -e .
-          pip install pytest pytest-cov
+          # Every extra, so every backend's tests run. The minimal job runs without any.
+          pip install -e ".[all,test]"
       - name: Run pytest with coverage
         run: python -m pytest tests/ -v --tb=short --cov=automation_file --cov-report=term-missing --cov-report=xml
       - name: Upload coverage artifact
@@ -66,3 +65,75 @@ jobs:
         with:
           name: coverage-xml
           path: coverage.xml
+
+  minimal:
+    # The base install: no cloud SDK and no GUI toolkit. The tests of a backend whose extra is
+    # missing skip; everything else, the import guard included, has to pass.
+    needs: lint
+    runs-on: windows-latest
+    timeout-minutes: 15  # no run yet: the floor of 15 minutes, to revisit after the first runs
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install the base package
+        run: |
+          python -m pip install --upgrade pip wheel
+          Copy-Item stable.toml pyproject.toml -Force
+          pip install -e ".[test]"
+      - name: Run pytest
+        run: python -m pytest tests/ -v --tb=short
+
+  extras:
+    # Each extra on its own: it installs, and its backend's tests run with no other SDK present.
+    needs: lint
+    runs-on: windows-latest
+    timeout-minutes: 15  # no run yet: the floor of 15 minutes, to revisit after the first runs
+    strategy:
+      fail-fast: false
+      matrix:
+        extra: [ s3, azure, gdrive, dropbox, sftp, ftp, webdav, smb, fsspec, onedrive, box, parquet, gui ]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install the package with one extra
+        run: |
+          python -m pip install --upgrade pip wheel
+          Copy-Item stable.toml pyproject.toml -Force
+          pip install -e ".[${{ matrix.extra }},test]"
+      - name: Run pytest
+        run: python -m pytest tests/ -v --tb=short
+
+  package:
+    # The sdist and the wheel build from stable.toml and their metadata renders, before anything is
+    # merged. It gates nothing: the publish job builds again from the locked tools.
+    needs: lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 15  # no run yet: the floor of 15 minutes, to revisit after the first runs
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Build the distribution and check its metadata
+        run: |
+          # The locked, wheels-only tools the publish jobs build with.
+          python -m pip install --require-hashes --only-binary :all: -r .github/requirements/publish.txt
+          cp stable.toml pyproject.toml
+          python -m build --no-isolation
+          twine check dist/*

.github/workflows/integration.yml

@@ -0,0 +1,87 @@
+name: Integration
+
+# The storage contract suite against real services, and the unit tests on the other two platforms.
+# Neither job gates a release: publishing depends on the jobs of ci-dev.yml and ci-stable.yml.
+
+on:
+  pull_request:
+    branches: [ "dev", "main" ]
+  push:
+    branches: [ "dev" ]
+  schedule:
+    - cron: "30 3 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: integration-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  services:
+    # One service per job, each started in a container by tests/integration/start_service.sh.
+    # FA_IT_REQUIRED turns a test module that would skip (its variables are missing) into an error.
+    runs-on: ubuntu-latest
+    timeout-minutes: 20  # no run yet: to revisit after the first runs
+    strategy:
+      fail-fast: false
+      matrix:
+        service: [ s3, azure, sftp, ftp, webdav, smb ]
+    env:
+      SERVICE: ${{ matrix.service }}
+      FA_IT_REQUIRED: "1"
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install the package with every extra
+        run: |
+          python -m pip install --require-hashes --only-binary :all: -r .github/requirements/integration.txt
+          cp dev.toml pyproject.toml
+          python -m pip install --no-deps --no-build-isolation --only-binary :all: -e .
+      - name: Start the service
+        run: bash tests/integration/start_service.sh "$SERVICE"
+      - name: Run the contract suite against it
+        run: python -m pytest tests/integration/test_"$SERVICE"_*.py -v --tb=short
+      - name: Show the service log
+        if: failure()
+        run: docker logs "fa-it-$SERVICE" || true
+
+  platforms:
+    # ci-dev.yml and ci-stable.yml run the unit tests on Windows. These are the other two.
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20  # no run yet: to revisit after the first runs
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, macos-latest ]
+    env:
+      QT_QPA_PLATFORM: offscreen
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install the libraries Qt needs without a display
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends libegl1 libgl1 libxkbcommon0 libdbus-1-3 libfontconfig1
+      - name: Install the package with every extra
+        run: |
+          python -m pip install --require-hashes --only-binary :all: -r .github/requirements/integration.txt
+          cp dev.toml pyproject.toml
+          python -m pip install --no-deps --no-build-isolation --only-binary :all: -e .
+      - name: Run pytest
+        run: python -m pytest tests/ -v --tb=short

.github/workflows/publish.yml

@@ -33,39 +33,11 @@ jobs:
         run: |
           python -m pip install --require-hashes --only-binary :all: -r .github/requirements/publish.txt
 
-      - name: Bump patch version in stable.toml and dev.toml
+      # A patch release unless the pull request raised MAJOR or MINOR in stable.toml and dev.toml.
+      # scripts/stable_release.py says how the two are told apart.
+      - name: Pick the release version in stable.toml and dev.toml
         id: bump
-        run: |
-          python <<'EOF'
-          import os
-          import pathlib
-          import re
-
-          def bump(path: pathlib.Path) -> str:
-              text = path.read_text(encoding="utf-8")
-              match = re.search(r'^version\s*=\s*"(\d+)\.(\d+)\.(\d+)"', text, re.MULTILINE)
-              if not match:
-                  raise SystemExit(f"Could not find version in {path}")
-              major, minor, patch = (int(part) for part in match.groups())
-              new_version = f"{major}.{minor}.{patch + 1}"
-              new_text = re.sub(
-                  r'^(version\s*=\s*)"\d+\.\d+\.\d+"',
-                  rf'\1"{new_version}"',
-                  text,
-                  count=1,
-                  flags=re.MULTILINE,
-              )
-              path.write_text(new_text, encoding="utf-8")
-              return new_version
-
-          stable_version = bump(pathlib.Path("stable.toml"))
-          dev_version = bump(pathlib.Path("dev.toml"))
-          with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output:
-              output.write(f"new_version={stable_version}\n")
-              output.write(f"dev_version={dev_version}\n")
-          print(f"stable.toml -> {stable_version}")
-          print(f"dev.toml    -> {dev_version}")
-          EOF
+        run: python scripts/stable_release.py bump
 
       - name: Use stable.toml as pyproject.toml
         run: cp stable.toml pyproject.toml