LLM/diffusers
1
0
Fork 0
mirror of https://github.com/huggingface/diffusers.git synced 2025-12-06 12:34:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

[ci] revisit the installations in CI. (#12450)

Browse Source 
* revisit the installations in CI.

* up

* up

* up

* empty

* up

* up

* up
This commit is contained in:
Sayak Paul
2025-10-08 19:21:24 +05:30
committed by GitHub
parent 345864eb85
commit a519272d97
20 changed files with 148 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/benchmark.yml vendored
View File

@@ -38,9 +38,8 @@ jobs:
run: | run: |
apt update apt update
apt install -y libpq-dev postgresql-client apt install -y libpq-dev postgresql-client
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test] uv pip install -r benchmarks/requirements.txt
python -m uv pip install -r benchmarks/requirements.txt
- name: Environment - name: Environment
run: | run: |
python utils/print_env.py python utils/print_env.py

2
.github/workflows/mirror_community_pipeline.yml vendored
View File

@@ -74,7 +74,7 @@ jobs:
python-version: "3.10" python-version: "3.10"
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m pip install --upgrade pip pip install --upgrade pip
pip install --upgrade huggingface_hub pip install --upgrade huggingface_hub
# Check secret is set # Check secret is set

91
.github/workflows/nightly_tests.yml vendored
View File

@@ -71,10 +71,9 @@ jobs:
run: nvidia-smi run: nvidia-smi
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test] uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip install pytest-reportlog
python -m uv pip install pytest-reportlog
- name: Environment - name: Environment
run: | run: |
python utils/print_env.py python utils/print_env.py
@@ -84,7 +83,7 @@ jobs:
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
CUBLAS_WORKSPACE_CONFIG: :16:8 CUBLAS_WORKSPACE_CONFIG: :16:8
run: | run: |
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ pytest -n 1 --max-worker-restart=0 --dist=loadfile \
-s -v -k "not Flax and not Onnx" \ -s -v -k "not Flax and not Onnx" \
--make-reports=tests_pipeline_${{ matrix.module }}_cuda \ --make-reports=tests_pipeline_${{ matrix.module }}_cuda \
--report-log=tests_pipeline_${{ matrix.module }}_cuda.log \ --report-log=tests_pipeline_${{ matrix.module }}_cuda.log \
@@ -124,11 +123,10 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test] uv pip install peft@git+https://github.com/huggingface/peft.git
python -m uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip install pytest-reportlog
python -m uv pip install pytest-reportlog
- name: Environment - name: Environment
run: python utils/print_env.py run: python utils/print_env.py
@@ -139,7 +137,7 @@ jobs:
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
CUBLAS_WORKSPACE_CONFIG: :16:8 CUBLAS_WORKSPACE_CONFIG: :16:8
run: | run: |
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ pytest -n 1 --max-worker-restart=0 --dist=loadfile \
-s -v -k "not Flax and not Onnx" \ -s -v -k "not Flax and not Onnx" \
--make-reports=tests_torch_${{ matrix.module }}_cuda \ --make-reports=tests_torch_${{ matrix.module }}_cuda \
--report-log=tests_torch_${{ matrix.module }}_cuda.log \ --report-log=tests_torch_${{ matrix.module }}_cuda.log \
@@ -152,7 +150,7 @@ jobs:
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
CUBLAS_WORKSPACE_CONFIG: :16:8 CUBLAS_WORKSPACE_CONFIG: :16:8
run: | run: |
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ pytest -n 1 --max-worker-restart=0 --dist=loadfile \
-s -v --make-reports=examples_torch_cuda \ -s -v --make-reports=examples_torch_cuda \
--report-log=examples_torch_cuda.log \ --report-log=examples_torch_cuda.log \
examples/ examples/
@@ -191,8 +189,7 @@ jobs:
nvidia-smi nvidia-smi
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality,training]"
python -m uv pip install -e [quality,test,training]
- name: Environment - name: Environment
run: | run: |
python utils/print_env.py python utils/print_env.py
@@ -201,7 +198,7 @@ jobs:
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
RUN_COMPILE: yes RUN_COMPILE: yes
run: | run: |
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "compile" --make-reports=tests_torch_compile_cuda tests/ pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "compile" --make-reports=tests_torch_compile_cuda tests/
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
run: cat reports/tests_torch_compile_cuda_failures_short.txt run: cat reports/tests_torch_compile_cuda_failures_short.txt
@@ -232,11 +229,10 @@ jobs:
run: nvidia-smi run: nvidia-smi
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test] uv pip install peft@git+https://github.com/huggingface/peft.git
python -m uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip install pytest-reportlog
python -m uv pip install pytest-reportlog
- name: Environment - name: Environment
run: | run: |
python utils/print_env.py python utils/print_env.py
@@ -247,7 +243,7 @@ jobs:
CUBLAS_WORKSPACE_CONFIG: :16:8 CUBLAS_WORKSPACE_CONFIG: :16:8
BIG_GPU_MEMORY: 40 BIG_GPU_MEMORY: 40
run: | run: |
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ pytest -n 1 --max-worker-restart=0 --dist=loadfile \
-m "big_accelerator" \ -m "big_accelerator" \
--make-reports=tests_big_gpu_torch_cuda \ --make-reports=tests_big_gpu_torch_cuda \
--report-log=tests_big_gpu_torch_cuda.log \ --report-log=tests_big_gpu_torch_cuda.log \
@@ -282,10 +278,9 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test] uv pip install peft@git+https://github.com/huggingface/peft.git
python -m uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
- name: Environment - name: Environment
run: | run: |
@@ -297,7 +292,7 @@ jobs:
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
CUBLAS_WORKSPACE_CONFIG: :16:8 CUBLAS_WORKSPACE_CONFIG: :16:8
run: | run: |
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ pytest -n 1 --max-worker-restart=0 --dist=loadfile \
-s -v -k "not Flax and not Onnx" \ -s -v -k "not Flax and not Onnx" \
--make-reports=tests_torch_minimum_version_cuda \ --make-reports=tests_torch_minimum_version_cuda \
tests/models/test_modeling_common.py \ tests/models/test_modeling_common.py \
@@ -357,13 +352,12 @@ jobs:
run: nvidia-smi run: nvidia-smi
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test] uv pip install -U ${{ matrix.config.backend }}
python -m uv pip install -U ${{ matrix.config.backend }}
if [ "${{ join(matrix.config.additional_deps, ' ') }}" != "" ]; then if [ "${{ join(matrix.config.additional_deps, ' ') }}" != "" ]; then
python -m uv pip install ${{ join(matrix.config.additional_deps, ' ') }} uv pip install ${{ join(matrix.config.additional_deps, ' ') }}
fi fi
python -m uv pip install pytest-reportlog uv pip install pytest-reportlog
- name: Environment - name: Environment
run: | run: |
python utils/print_env.py python utils/print_env.py
@@ -374,7 +368,7 @@ jobs:
CUBLAS_WORKSPACE_CONFIG: :16:8 CUBLAS_WORKSPACE_CONFIG: :16:8
BIG_GPU_MEMORY: 40 BIG_GPU_MEMORY: 40
run: | run: |
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ pytest -n 1 --max-worker-restart=0 --dist=loadfile \
--make-reports=tests_${{ matrix.config.backend }}_torch_cuda \ --make-reports=tests_${{ matrix.config.backend }}_torch_cuda \
--report-log=tests_${{ matrix.config.backend }}_torch_cuda.log \ --report-log=tests_${{ matrix.config.backend }}_torch_cuda.log \
tests/quantization/${{ matrix.config.test_location }} tests/quantization/${{ matrix.config.test_location }}
@@ -409,10 +403,9 @@ jobs:
run: nvidia-smi run: nvidia-smi
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test] uv pip install -U bitsandbytes optimum_quanto
python -m uv pip install -U bitsandbytes optimum_quanto uv pip install pytest-reportlog
python -m uv pip install pytest-reportlog
- name: Environment - name: Environment
run: | run: |
python utils/print_env.py python utils/print_env.py
@@ -423,7 +416,7 @@ jobs:
CUBLAS_WORKSPACE_CONFIG: :16:8 CUBLAS_WORKSPACE_CONFIG: :16:8
BIG_GPU_MEMORY: 40 BIG_GPU_MEMORY: 40
run: | run: |
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ pytest -n 1 --max-worker-restart=0 --dist=loadfile \
--make-reports=tests_pipeline_level_quant_torch_cuda \ --make-reports=tests_pipeline_level_quant_torch_cuda \
--report-log=tests_pipeline_level_quant_torch_cuda.log \ --report-log=tests_pipeline_level_quant_torch_cuda.log \
tests/quantization/test_pipeline_level_quantization.py tests/quantization/test_pipeline_level_quantization.py
@@ -523,11 +516,11 @@ jobs:
# - name: Install dependencies # - name: Install dependencies
# shell: arch -arch arm64 bash {0} # shell: arch -arch arm64 bash {0}
# run: | # run: |
# ${CONDA_RUN} python -m pip install --upgrade pip uv # ${CONDA_RUN} pip install --upgrade pip uv
# ${CONDA_RUN} python -m uv pip install -e [quality,test] # ${CONDA_RUN} uv pip install -e ".[quality]"
# ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu # ${CONDA_RUN} uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
# ${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate # ${CONDA_RUN} uv pip install accelerate@git+https://github.com/huggingface/accelerate
# ${CONDA_RUN} python -m uv pip install pytest-reportlog # ${CONDA_RUN} uv pip install pytest-reportlog
# - name: Environment # - name: Environment
# shell: arch -arch arm64 bash {0} # shell: arch -arch arm64 bash {0}
# run: | # run: |
@@ -538,7 +531,7 @@ jobs:
# HF_HOME: /System/Volumes/Data/mnt/cache # HF_HOME: /System/Volumes/Data/mnt/cache
# HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} # HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
# run: | # run: |
# ${CONDA_RUN} python -m pytest -n 1 -s -v --make-reports=tests_torch_mps \ # ${CONDA_RUN} pytest -n 1 -s -v --make-reports=tests_torch_mps \
# --report-log=tests_torch_mps.log \ # --report-log=tests_torch_mps.log \
# tests/ # tests/
# - name: Failure short reports # - name: Failure short reports
@@ -579,11 +572,11 @@ jobs:
# - name: Install dependencies # - name: Install dependencies
# shell: arch -arch arm64 bash {0} # shell: arch -arch arm64 bash {0}
# run: | # run: |
# ${CONDA_RUN} python -m pip install --upgrade pip uv # ${CONDA_RUN} pip install --upgrade pip uv
# ${CONDA_RUN} python -m uv pip install -e [quality,test] # ${CONDA_RUN} uv pip install -e ".[quality]"
# ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu # ${CONDA_RUN} uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
# ${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate # ${CONDA_RUN} uv pip install accelerate@git+https://github.com/huggingface/accelerate
# ${CONDA_RUN} python -m uv pip install pytest-reportlog # ${CONDA_RUN} uv pip install pytest-reportlog
# - name: Environment # - name: Environment
# shell: arch -arch arm64 bash {0} # shell: arch -arch arm64 bash {0}
# run: | # run: |
@@ -594,7 +587,7 @@ jobs:
# HF_HOME: /System/Volumes/Data/mnt/cache # HF_HOME: /System/Volumes/Data/mnt/cache
# HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} # HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
# run: | # run: |
# ${CONDA_RUN} python -m pytest -n 1 -s -v --make-reports=tests_torch_mps \ # ${CONDA_RUN} pytest -n 1 -s -v --make-reports=tests_torch_mps \
# --report-log=tests_torch_mps.log \ # --report-log=tests_torch_mps.log \
# tests/ # tests/
# - name: Failure short reports # - name: Failure short reports

9
.github/workflows/pr_dependency_test.yml vendored
View File

@@ -25,11 +25,8 @@ jobs:
python-version: "3.8" python-version: "3.8"
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" pip install -e .
python -m pip install --upgrade pip uv pip install pytest
python -m uv pip install -e .
python -m uv pip install pytest
- name: Check for soft dependencies - name: Check for soft dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" pytest tests/others/test_dependencies.py
pytest tests/others/test_dependencies.py

15
.github/workflows/pr_modular_tests.yml vendored
View File

@@ -42,7 +42,7 @@ jobs:
python-version: "3.10" python-version: "3.10"
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m pip install --upgrade pip pip install --upgrade pip
pip install .[quality] pip install .[quality]
- name: Check quality - name: Check quality
run: make quality run: make quality
@@ -62,7 +62,7 @@ jobs:
python-version: "3.10" python-version: "3.10"
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m pip install --upgrade pip pip install --upgrade pip
pip install .[quality] pip install .[quality]
- name: Check repo consistency - name: Check repo consistency
run: | run: |
@@ -108,21 +108,18 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test] uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
pip uninstall transformers -y && pip uninstall huggingface_hub -y && python -m uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps
- name: Environment - name: Environment
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
python utils/print_env.py python utils/print_env.py
- name: Run fast PyTorch Pipeline CPU tests - name: Run fast PyTorch Pipeline CPU tests
if: ${{ matrix.config.framework == 'pytorch_pipelines' }} if: ${{ matrix.config.framework == 'pytorch_pipelines' }}
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" pytest -n 8 --max-worker-restart=0 --dist=loadfile \
python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile \
-s -v -k "not Flax and not Onnx" \ -s -v -k "not Flax and not Onnx" \
--make-reports=tests_${{ matrix.config.report }} \ --make-reports=tests_${{ matrix.config.report }} \
tests/modular_pipelines tests/modular_pipelines

19
.github/workflows/pr_test_fetcher.yml vendored
View File

@@ -33,8 +33,7 @@ jobs:
fetch-depth: 0 fetch-depth: 0
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test]
- name: Environment - name: Environment
run: | run: |
python utils/print_env.py python utils/print_env.py
@@ -90,19 +89,16 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m pip install -e [quality,test] uv pip install accelerate
python -m pip install accelerate
- name: Environment - name: Environment
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
python utils/print_env.py python utils/print_env.py
- name: Run all selected tests on CPU - name: Run all selected tests on CPU
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" pytest -n 2 --dist=loadfile -v --make-reports=${{ matrix.modules }}_tests_cpu ${{ fromJson(needs.setup_pr_tests.outputs.test_map)[matrix.modules] }}
python -m pytest -n 2 --dist=loadfile -v --make-reports=${{ matrix.modules }}_tests_cpu ${{ fromJson(needs.setup_pr_tests.outputs.test_map)[matrix.modules] }}
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
@@ -148,19 +144,16 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" pip install -e [quality]
python -m pip install -e [quality,test]
- name: Environment - name: Environment
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
python utils/print_env.py python utils/print_env.py
- name: Run Hub tests for models, schedulers, and pipelines on a staging env - name: Run Hub tests for models, schedulers, and pipelines on a staging env
if: ${{ matrix.config.framework == 'hub_tests_pytorch' }} if: ${{ matrix.config.framework == 'hub_tests_pytorch' }}
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" HUGGINGFACE_CO_STAGING=true pytest \
HUGGINGFACE_CO_STAGING=true python -m pytest \
-m "is_staging_test" \ -m "is_staging_test" \
--make-reports=tests_${{ matrix.config.report }} \ --make-reports=tests_${{ matrix.config.report }} \
tests tests

48
.github/workflows/pr_tests.yml vendored
View File

@@ -38,7 +38,7 @@ jobs:
python-version: "3.8" python-version: "3.8"
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m pip install --upgrade pip pip install --upgrade pip
pip install .[quality] pip install .[quality]
- name: Check quality - name: Check quality
run: make quality run: make quality
@@ -58,7 +58,7 @@ jobs:
python-version: "3.8" python-version: "3.8"
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m pip install --upgrade pip pip install --upgrade pip
pip install .[quality] pip install .[quality]
- name: Check repo consistency - name: Check repo consistency
run: | run: |
@@ -114,21 +114,18 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test] uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
pip uninstall transformers -y && pip uninstall huggingface_hub -y && python -m uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps
- name: Environment - name: Environment
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
python utils/print_env.py python utils/print_env.py
- name: Run fast PyTorch Pipeline CPU tests - name: Run fast PyTorch Pipeline CPU tests
if: ${{ matrix.config.framework == 'pytorch_pipelines' }} if: ${{ matrix.config.framework == 'pytorch_pipelines' }}
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" pytest -n 8 --max-worker-restart=0 --dist=loadfile \
python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile \
-s -v -k "not Flax and not Onnx" \ -s -v -k "not Flax and not Onnx" \
--make-reports=tests_${{ matrix.config.report }} \ --make-reports=tests_${{ matrix.config.report }} \
tests/pipelines tests/pipelines
@@ -136,8 +133,7 @@ jobs:
- name: Run fast PyTorch Model Scheduler CPU tests - name: Run fast PyTorch Model Scheduler CPU tests
if: ${{ matrix.config.framework == 'pytorch_models' }} if: ${{ matrix.config.framework == 'pytorch_models' }}
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" pytest -n 4 --max-worker-restart=0 --dist=loadfile \
python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile \
-s -v -k "not Flax and not Onnx and not Dependency" \ -s -v -k "not Flax and not Onnx and not Dependency" \
--make-reports=tests_${{ matrix.config.report }} \ --make-reports=tests_${{ matrix.config.report }} \
tests/models tests/schedulers tests/others tests/models tests/schedulers tests/others
@@ -145,9 +141,8 @@ jobs:
- name: Run example PyTorch CPU tests - name: Run example PyTorch CPU tests
if: ${{ matrix.config.framework == 'pytorch_examples' }} if: ${{ matrix.config.framework == 'pytorch_examples' }}
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install ".[training]"
python -m uv pip install peft timm pytest -n 4 --max-worker-restart=0 --dist=loadfile \
python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile \
--make-reports=tests_${{ matrix.config.report }} \ --make-reports=tests_${{ matrix.config.report }} \
examples examples
@@ -195,19 +190,16 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test]
- name: Environment - name: Environment
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
python utils/print_env.py python utils/print_env.py
- name: Run Hub tests for models, schedulers, and pipelines on a staging env - name: Run Hub tests for models, schedulers, and pipelines on a staging env
if: ${{ matrix.config.framework == 'hub_tests_pytorch' }} if: ${{ matrix.config.framework == 'hub_tests_pytorch' }}
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" HUGGINGFACE_CO_STAGING=true pytest \
HUGGINGFACE_CO_STAGING=true python -m pytest \
-m "is_staging_test" \ -m "is_staging_test" \
--make-reports=tests_${{ matrix.config.report }} \ --make-reports=tests_${{ matrix.config.report }} \
tests tests
@@ -249,27 +241,24 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test]
# TODO (sayakpaul, DN6): revisit `--no-deps` # TODO (sayakpaul, DN6): revisit `--no-deps`
python -m pip install -U peft@git+https://github.com/huggingface/peft.git --no-deps uv pip install -U peft@git+https://github.com/huggingface/peft.git --no-deps
python -m uv pip install -U tokenizers uv pip install -U tokenizers
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps
pip uninstall transformers -y && pip uninstall huggingface_hub -y && python -m uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
- name: Environment - name: Environment
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
python utils/print_env.py python utils/print_env.py
- name: Run fast PyTorch LoRA tests with PEFT - name: Run fast PyTorch LoRA tests with PEFT
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" pytest -n 4 --max-worker-restart=0 --dist=loadfile \
python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile \
-s -v \ -s -v \
--make-reports=tests_peft_main \ --make-reports=tests_peft_main \
tests/lora/ tests/lora/
python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile \ pytest -n 4 --max-worker-restart=0 --dist=loadfile \
-s -v \ -s -v \
--make-reports=tests_models_lora_peft_main \ --make-reports=tests_models_lora_peft_main \
tests/models/ -k "lora" tests/models/ -k "lora"
@@ -286,3 +275,4 @@ jobs:
with: with:
name: pr_main_test_reports name: pr_main_test_reports
path: reports path: reports

42
.github/workflows/pr_tests_gpu.yml vendored
View File

@@ -39,7 +39,7 @@ jobs:
python-version: "3.8" python-version: "3.8"
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m pip install --upgrade pip pip install --upgrade pip
pip install .[quality] pip install .[quality]
- name: Check quality - name: Check quality
run: make quality run: make quality
@@ -59,7 +59,7 @@ jobs:
python-version: "3.8" python-version: "3.8"
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m pip install --upgrade pip pip install --upgrade pip
pip install .[quality] pip install .[quality]
- name: Check repo consistency - name: Check repo consistency
run: | run: |
@@ -88,8 +88,7 @@ jobs:
fetch-depth: 2 fetch-depth: 2
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test]
- name: Environment - name: Environment
run: | run: |
python utils/print_env.py python utils/print_env.py
@@ -130,10 +129,9 @@ jobs:
nvidia-smi nvidia-smi
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test] uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
pip uninstall transformers -y && pip uninstall huggingface_hub -y && python -m uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
- name: Environment - name: Environment
run: | run: |
@@ -152,13 +150,13 @@ jobs:
CUBLAS_WORKSPACE_CONFIG: :16:8 CUBLAS_WORKSPACE_CONFIG: :16:8
run: | run: |
if [ "${{ matrix.module }}" = "ip_adapters" ]; then if [ "${{ matrix.module }}" = "ip_adapters" ]; then
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ pytest -n 1 --max-worker-restart=0 --dist=loadfile \
-s -v -k "not Flax and not Onnx" \ -s -v -k "not Flax and not Onnx" \
--make-reports=tests_pipeline_${{ matrix.module }}_cuda \ --make-reports=tests_pipeline_${{ matrix.module }}_cuda \
tests/pipelines/${{ matrix.module }} tests/pipelines/${{ matrix.module }}
else else
pattern=$(cat ${{ steps.extract_tests.outputs.pattern_file }}) pattern=$(cat ${{ steps.extract_tests.outputs.pattern_file }})
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ pytest -n 1 --max-worker-restart=0 --dist=loadfile \
-s -v -k "not Flax and not Onnx and $pattern" \ -s -v -k "not Flax and not Onnx and $pattern" \
--make-reports=tests_pipeline_${{ matrix.module }}_cuda \ --make-reports=tests_pipeline_${{ matrix.module }}_cuda \
tests/pipelines/${{ matrix.module }} tests/pipelines/${{ matrix.module }}
@@ -200,11 +198,10 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test] uv pip install peft@git+https://github.com/huggingface/peft.git
python -m uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
pip uninstall transformers -y && pip uninstall huggingface_hub -y && python -m uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
- name: Environment - name: Environment
run: | run: |
@@ -225,10 +222,10 @@ jobs:
run: | run: |
pattern=$(cat ${{ steps.extract_tests.outputs.pattern_file }}) pattern=$(cat ${{ steps.extract_tests.outputs.pattern_file }})
if [ -z "$pattern" ]; then if [ -z "$pattern" ]; then
python -m pytest -n 1 -sv --max-worker-restart=0 --dist=loadfile -k "not Flax and not Onnx" tests/${{ matrix.module }} \ pytest -n 1 -sv --max-worker-restart=0 --dist=loadfile -k "not Flax and not Onnx" tests/${{ matrix.module }} \
--make-reports=tests_torch_cuda_${{ matrix.module }} --make-reports=tests_torch_cuda_${{ matrix.module }}
else else
python -m pytest -n 1 -sv --max-worker-restart=0 --dist=loadfile -k "not Flax and not Onnx and $pattern" tests/${{ matrix.module }} \ pytest -n 1 -sv --max-worker-restart=0 --dist=loadfile -k "not Flax and not Onnx and $pattern" tests/${{ matrix.module }} \
--make-reports=tests_torch_cuda_${{ matrix.module }} --make-reports=tests_torch_cuda_${{ matrix.module }}
fi fi
@@ -265,22 +262,19 @@ jobs:
nvidia-smi nvidia-smi
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
pip uninstall transformers -y && pip uninstall huggingface_hub -y && python -m uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git uv pip install -e ".[quality,training]"
python -m uv pip install -e [quality,test,training]
- name: Environment - name: Environment
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
python utils/print_env.py python utils/print_env.py
- name: Run example tests on GPU - name: Run example tests on GPU
env: env:
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install ".[training]"
python -m uv pip install timm pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_cuda examples/
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_cuda examples/
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}

10
.github/workflows/pr_torch_dependency_test.yml vendored
View File

@@ -25,12 +25,8 @@ jobs:
python-version: "3.8" python-version: "3.8"
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" pip install -e .
python -m pip install --upgrade pip uv pip install torch torchvision torchaudio pytest
python -m uv pip install -e .
python -m uv pip install torch torchvision torchaudio
python -m uv pip install pytest
- name: Check for soft dependencies - name: Check for soft dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" pytest tests/others/test_dependencies.py
pytest tests/others/test_dependencies.py

38
.github/workflows/push_tests.yml vendored
View File

@@ -34,8 +34,7 @@ jobs:
fetch-depth: 2 fetch-depth: 2
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test]
- name: Environment - name: Environment
run: | run: |
python utils/print_env.py python utils/print_env.py
@@ -75,9 +74,8 @@ jobs:
nvidia-smi nvidia-smi
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test] uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
- name: Environment - name: Environment
run: | run: |
python utils/print_env.py python utils/print_env.py
@@ -87,7 +85,7 @@ jobs:
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
CUBLAS_WORKSPACE_CONFIG: :16:8 CUBLAS_WORKSPACE_CONFIG: :16:8
run: | run: |
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ pytest -n 1 --max-worker-restart=0 --dist=loadfile \
-s -v -k "not Flax and not Onnx" \ -s -v -k "not Flax and not Onnx" \
--make-reports=tests_pipeline_${{ matrix.module }}_cuda \ --make-reports=tests_pipeline_${{ matrix.module }}_cuda \
tests/pipelines/${{ matrix.module }} tests/pipelines/${{ matrix.module }}
@@ -126,10 +124,9 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test] uv pip install peft@git+https://github.com/huggingface/peft.git
python -m uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
- name: Environment - name: Environment
run: | run: |
@@ -141,7 +138,7 @@ jobs:
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
CUBLAS_WORKSPACE_CONFIG: :16:8 CUBLAS_WORKSPACE_CONFIG: :16:8
run: | run: |
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ pytest -n 1 --max-worker-restart=0 --dist=loadfile \
-s -v -k "not Flax and not Onnx" \ -s -v -k "not Flax and not Onnx" \
--make-reports=tests_torch_cuda_${{ matrix.module }} \ --make-reports=tests_torch_cuda_${{ matrix.module }} \
tests/${{ matrix.module }} tests/${{ matrix.module }}
@@ -180,8 +177,7 @@ jobs:
nvidia-smi nvidia-smi
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality,training]"
python -m uv pip install -e [quality,test,training]
- name: Environment - name: Environment
run: | run: |
python utils/print_env.py python utils/print_env.py
@@ -190,7 +186,7 @@ jobs:
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
RUN_COMPILE: yes RUN_COMPILE: yes
run: | run: |
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "compile" --make-reports=tests_torch_compile_cuda tests/ pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "compile" --make-reports=tests_torch_compile_cuda tests/
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
run: cat reports/tests_torch_compile_cuda_failures_short.txt run: cat reports/tests_torch_compile_cuda_failures_short.txt
@@ -223,8 +219,7 @@ jobs:
nvidia-smi nvidia-smi
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality,training]"
python -m uv pip install -e [quality,test,training]
- name: Environment - name: Environment
run: | run: |
python utils/print_env.py python utils/print_env.py
@@ -232,7 +227,7 @@ jobs:
env: env:
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
run: | run: |
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "xformers" --make-reports=tests_torch_xformers_cuda tests/ pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "xformers" --make-reports=tests_torch_xformers_cuda tests/
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
run: cat reports/tests_torch_xformers_cuda_failures_short.txt run: cat reports/tests_torch_xformers_cuda_failures_short.txt
@@ -264,21 +259,18 @@ jobs:
nvidia-smi nvidia-smi
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality,training]"
python -m uv pip install -e [quality,test,training]
- name: Environment - name: Environment
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
python utils/print_env.py python utils/print_env.py
- name: Run example tests on GPU - name: Run example tests on GPU
env: env:
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install ".[training]"
python -m uv pip install timm pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_cuda examples/
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_cuda examples/
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}

12
.github/workflows/push_tests_fast.yml vendored
View File

@@ -60,19 +60,16 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test]
- name: Environment - name: Environment
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
python utils/print_env.py python utils/print_env.py
- name: Run fast PyTorch CPU tests - name: Run fast PyTorch CPU tests
if: ${{ matrix.config.framework == 'pytorch' }} if: ${{ matrix.config.framework == 'pytorch' }}
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" pytest -n 4 --max-worker-restart=0 --dist=loadfile \
python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile \
-s -v -k "not Flax and not Onnx" \ -s -v -k "not Flax and not Onnx" \
--make-reports=tests_${{ matrix.config.report }} \ --make-reports=tests_${{ matrix.config.report }} \
tests/ tests/
@@ -80,9 +77,8 @@ jobs:
- name: Run example PyTorch CPU tests - name: Run example PyTorch CPU tests
if: ${{ matrix.config.framework == 'pytorch_examples' }} if: ${{ matrix.config.framework == 'pytorch_examples' }}
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install ".[training]"
python -m uv pip install peft timm pytest -n 4 --max-worker-restart=0 --dist=loadfile \
python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile \
--make-reports=tests_${{ matrix.config.report }} \ --make-reports=tests_${{ matrix.config.report }} \
examples examples

47
.github/workflows/release_tests_fast.yml vendored
View File

@@ -32,8 +32,7 @@ jobs:
fetch-depth: 2 fetch-depth: 2
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test]
- name: Environment - name: Environment
run: | run: |
python utils/print_env.py python utils/print_env.py
@@ -73,9 +72,8 @@ jobs:
nvidia-smi nvidia-smi
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test] uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
- name: Environment - name: Environment
run: | run: |
python utils/print_env.py python utils/print_env.py
@@ -85,7 +83,7 @@ jobs:
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
CUBLAS_WORKSPACE_CONFIG: :16:8 CUBLAS_WORKSPACE_CONFIG: :16:8
run: | run: |
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ pytest -n 1 --max-worker-restart=0 --dist=loadfile \
-s -v -k "not Flax and not Onnx" \ -s -v -k "not Flax and not Onnx" \
--make-reports=tests_pipeline_${{ matrix.module }}_cuda \ --make-reports=tests_pipeline_${{ matrix.module }}_cuda \
tests/pipelines/${{ matrix.module }} tests/pipelines/${{ matrix.module }}
@@ -124,10 +122,9 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test] uv pip install peft@git+https://github.com/huggingface/peft.git
python -m uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
- name: Environment - name: Environment
run: | run: |
@@ -139,7 +136,7 @@ jobs:
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
CUBLAS_WORKSPACE_CONFIG: :16:8 CUBLAS_WORKSPACE_CONFIG: :16:8
run: | run: |
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ pytest -n 1 --max-worker-restart=0 --dist=loadfile \
-s -v -k "not Flax and not Onnx" \ -s -v -k "not Flax and not Onnx" \
--make-reports=tests_torch_${{ matrix.module }}_cuda \ --make-reports=tests_torch_${{ matrix.module }}_cuda \
tests/${{ matrix.module }} tests/${{ matrix.module }}
@@ -175,10 +172,9 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test] uv pip install peft@git+https://github.com/huggingface/peft.git
python -m uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
- name: Environment - name: Environment
run: | run: |
@@ -190,7 +186,7 @@ jobs:
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
CUBLAS_WORKSPACE_CONFIG: :16:8 CUBLAS_WORKSPACE_CONFIG: :16:8
run: | run: |
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ pytest -n 1 --max-worker-restart=0 --dist=loadfile \
-s -v -k "not Flax and not Onnx" \ -s -v -k "not Flax and not Onnx" \
--make-reports=tests_torch_minimum_cuda \ --make-reports=tests_torch_minimum_cuda \
tests/models/test_modeling_common.py \ tests/models/test_modeling_common.py \
@@ -235,8 +231,7 @@ jobs:
nvidia-smi nvidia-smi
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality,training]"
python -m uv pip install -e [quality,test,training]
- name: Environment - name: Environment
run: | run: |
python utils/print_env.py python utils/print_env.py
@@ -245,7 +240,7 @@ jobs:
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
RUN_COMPILE: yes RUN_COMPILE: yes
run: | run: |
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "compile" --make-reports=tests_torch_compile_cuda tests/ pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "compile" --make-reports=tests_torch_compile_cuda tests/
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
run: cat reports/tests_torch_compile_cuda_failures_short.txt run: cat reports/tests_torch_compile_cuda_failures_short.txt
@@ -278,8 +273,7 @@ jobs:
nvidia-smi nvidia-smi
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality,training]"
python -m uv pip install -e [quality,test,training]
- name: Environment - name: Environment
run: | run: |
python utils/print_env.py python utils/print_env.py
@@ -287,7 +281,7 @@ jobs:
env: env:
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
run: | run: |
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "xformers" --make-reports=tests_torch_xformers_cuda tests/ pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "xformers" --make-reports=tests_torch_xformers_cuda tests/
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
run: cat reports/tests_torch_xformers_cuda_failures_short.txt run: cat reports/tests_torch_xformers_cuda_failures_short.txt
@@ -321,21 +315,18 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality,training]"
python -m uv pip install -e [quality,test,training]
- name: Environment - name: Environment
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
python utils/print_env.py python utils/print_env.py
- name: Run example tests on GPU - name: Run example tests on GPU
env: env:
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install ".[training]"
python -m uv pip install timm pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_cuda examples/
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_cuda examples/
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}

5
.github/workflows/run_tests_from_a_pr.yml vendored
View File

@@ -63,9 +63,8 @@ jobs:
- name: Install pytest - name: Install pytest
run: | run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" uv pip install -e ".[quality]"
python -m uv pip install -e [quality,test] uv pip install peft
python -m uv pip install peft
- name: Run tests - name: Run tests
env: env:

1
docker/diffusers-doc-builder/Dockerfile
View File

@@ -11,6 +11,7 @@ RUN apt-get -y update && apt-get install -y bash \
git-lfs \ git-lfs \
curl \ curl \
ca-certificates \ ca-certificates \
libglib2.0-0 \
libsndfile1-dev \ libsndfile1-dev \
libgl1 libgl1

1
docker/diffusers-pytorch-cpu/Dockerfile
View File

@@ -11,6 +11,7 @@ RUN apt-get -y update && apt-get install -y bash \
git-lfs \ git-lfs \
curl \ curl \
ca-certificates \ ca-certificates \
libglib2.0-0 \
libsndfile1-dev \ libsndfile1-dev \
libgl1 libgl1

1
docker/diffusers-pytorch-cuda/Dockerfile
View File

@@ -16,6 +16,7 @@ RUN apt install -y bash \
git-lfs \ git-lfs \
curl \ curl \
ca-certificates \ ca-certificates \
libglib2.0-0 \
libsndfile1-dev \ libsndfile1-dev \
libgl1 \ libgl1 \
python3 \ python3 \

1
docker/diffusers-pytorch-minimum-cuda/Dockerfile
View File

@@ -19,6 +19,7 @@ RUN apt install -y bash \
git-lfs \ git-lfs \
curl \ curl \
ca-certificates \ ca-certificates \
libglib2.0-0 \
libsndfile1-dev \ libsndfile1-dev \
libgl1 \ libgl1 \
python3 \ python3 \

1
docker/diffusers-pytorch-xformers-cuda/Dockerfile
View File

@@ -16,6 +16,7 @@ RUN apt install -y bash \
git-lfs \ git-lfs \
curl \ curl \
ca-certificates \ ca-certificates \
libglib2.0-0 \
libsndfile1-dev \ libsndfile1-dev \
libgl1 \ libgl1 \
python3 \ python3 \

3
setup.py
View File

@@ -145,6 +145,7 @@ _deps = [
"black", "black",
"phonemizer", "phonemizer",
"opencv-python", "opencv-python",
"timm",
] ]
# this is a lookup table with items like: # this is a lookup table with items like:
@@ -218,7 +219,7 @@ class DepsTableUpdateCommand(Command):
extras = {} extras = {}
extras["quality"] = deps_list("urllib3", "isort", "ruff", "hf-doc-builder") extras["quality"] = deps_list("urllib3", "isort", "ruff", "hf-doc-builder")
extras["docs"] = deps_list("hf-doc-builder") extras["docs"] = deps_list("hf-doc-builder")
extras["training"] = deps_list("accelerate", "datasets", "protobuf", "tensorboard", "Jinja2", "peft") extras["training"] = deps_list("accelerate", "datasets", "protobuf", "tensorboard", "Jinja2", "peft", "timm")
extras["test"] = deps_list( extras["test"] = deps_list(
"compel", "compel",
"GitPython", "GitPython",

1
src/diffusers/dependency_versions_table.py
View File

@@ -52,4 +52,5 @@ deps = {
"black": "black", "black": "black",
"phonemizer": "phonemizer", "phonemizer": "phonemizer",
"opencv-python": "opencv-python", "opencv-python": "opencv-python",
"timm": "timm",
} }