mirror of
https://github.com/huggingface/diffusers.git
synced 2025-12-16 09:24:54 +08:00
Compare commits
2 Commits
animatelcm
...
v0.26.0
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
674d43fd68 | ||
|
|
e7a16666ea |
2
.github/workflows/benchmark.yml
vendored
2
.github/workflows/benchmark.yml
vendored
@@ -32,7 +32,7 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
apt-get update && apt-get install libsndfile1-dev libgl1 -y
|
apt-get update && apt-get install libsndfile1-dev libgl1 -y
|
||||||
python -m pip install -e .[quality,test]
|
python -m pip install -e .[quality,test]
|
||||||
python -m pip install pandas peft
|
python -m pip install pandas
|
||||||
- name: Environment
|
- name: Environment
|
||||||
run: |
|
run: |
|
||||||
python utils/print_env.py
|
python utils/print_env.py
|
||||||
|
|||||||
13
.github/workflows/pr_tests.yml
vendored
13
.github/workflows/pr_tests.yml
vendored
@@ -34,6 +34,11 @@ jobs:
|
|||||||
runner: docker-cpu
|
runner: docker-cpu
|
||||||
image: diffusers/diffusers-pytorch-cpu
|
image: diffusers/diffusers-pytorch-cpu
|
||||||
report: torch_cpu_models_schedulers
|
report: torch_cpu_models_schedulers
|
||||||
|
- name: LoRA
|
||||||
|
framework: lora
|
||||||
|
runner: docker-cpu
|
||||||
|
image: diffusers/diffusers-pytorch-cpu
|
||||||
|
report: torch_cpu_lora
|
||||||
- name: Fast Flax CPU tests
|
- name: Fast Flax CPU tests
|
||||||
framework: flax
|
framework: flax
|
||||||
runner: docker-cpu
|
runner: docker-cpu
|
||||||
@@ -89,6 +94,14 @@ jobs:
|
|||||||
--make-reports=tests_${{ matrix.config.report }} \
|
--make-reports=tests_${{ matrix.config.report }} \
|
||||||
tests/models tests/schedulers tests/others
|
tests/models tests/schedulers tests/others
|
||||||
|
|
||||||
|
- name: Run fast PyTorch LoRA CPU tests
|
||||||
|
if: ${{ matrix.config.framework == 'lora' }}
|
||||||
|
run: |
|
||||||
|
python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile \
|
||||||
|
-s -v -k "not Flax and not Onnx and not Dependency" \
|
||||||
|
--make-reports=tests_${{ matrix.config.report }} \
|
||||||
|
tests/lora
|
||||||
|
|
||||||
- name: Run fast Flax TPU tests
|
- name: Run fast Flax TPU tests
|
||||||
if: ${{ matrix.config.framework == 'flax' }}
|
if: ${{ matrix.config.framework == 'flax' }}
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -26,9 +26,9 @@ ENV PATH="/opt/venv/bin:$PATH"
|
|||||||
# pre-install the heavy dependencies (these can later be overridden by the deps from setup.py)
|
# pre-install the heavy dependencies (these can later be overridden by the deps from setup.py)
|
||||||
RUN python3.9 -m pip install --no-cache-dir --upgrade pip && \
|
RUN python3.9 -m pip install --no-cache-dir --upgrade pip && \
|
||||||
python3.9 -m pip install --no-cache-dir \
|
python3.9 -m pip install --no-cache-dir \
|
||||||
torch \
|
torch==2.1.2 \
|
||||||
torchvision \
|
torchvision==0.16.2 \
|
||||||
torchaudio \
|
torchaudio==2.1.2 \
|
||||||
invisible_watermark && \
|
invisible_watermark && \
|
||||||
python3.9 -m pip install --no-cache-dir \
|
python3.9 -m pip install --no-cache-dir \
|
||||||
accelerate \
|
accelerate \
|
||||||
|
|||||||
@@ -25,9 +25,9 @@ ENV PATH="/opt/venv/bin:$PATH"
|
|||||||
# pre-install the heavy dependencies (these can later be overridden by the deps from setup.py)
|
# pre-install the heavy dependencies (these can later be overridden by the deps from setup.py)
|
||||||
RUN python3 -m pip install --no-cache-dir --upgrade pip && \
|
RUN python3 -m pip install --no-cache-dir --upgrade pip && \
|
||||||
python3 -m pip install --no-cache-dir \
|
python3 -m pip install --no-cache-dir \
|
||||||
torch \
|
torch==2.1.2 \
|
||||||
torchvision \
|
torchvision==0.16.2 \
|
||||||
torchaudio \
|
torchaudio==2.1.2 \
|
||||||
invisible_watermark \
|
invisible_watermark \
|
||||||
--extra-index-url https://download.pytorch.org/whl/cpu && \
|
--extra-index-url https://download.pytorch.org/whl/cpu && \
|
||||||
python3 -m pip install --no-cache-dir \
|
python3 -m pip install --no-cache-dir \
|
||||||
|
|||||||
@@ -25,9 +25,9 @@ ENV PATH="/opt/venv/bin:$PATH"
|
|||||||
# pre-install the heavy dependencies (these can later be overridden by the deps from setup.py)
|
# pre-install the heavy dependencies (these can later be overridden by the deps from setup.py)
|
||||||
RUN python3 -m pip install --no-cache-dir --upgrade pip && \
|
RUN python3 -m pip install --no-cache-dir --upgrade pip && \
|
||||||
python3 -m pip install --no-cache-dir \
|
python3 -m pip install --no-cache-dir \
|
||||||
torch \
|
torch==2.1.2 \
|
||||||
torchvision \
|
torchvision==0.16.2 \
|
||||||
torchaudio \
|
torchaudio==2.1.2 \
|
||||||
invisible_watermark && \
|
invisible_watermark && \
|
||||||
python3 -m pip install --no-cache-dir \
|
python3 -m pip install --no-cache-dir \
|
||||||
accelerate \
|
accelerate \
|
||||||
|
|||||||
@@ -25,9 +25,9 @@ ENV PATH="/opt/venv/bin:$PATH"
|
|||||||
# pre-install the heavy dependencies (these can later be overridden by the deps from setup.py)
|
# pre-install the heavy dependencies (these can later be overridden by the deps from setup.py)
|
||||||
RUN python3 -m pip install --no-cache-dir --upgrade pip && \
|
RUN python3 -m pip install --no-cache-dir --upgrade pip && \
|
||||||
python3 -m pip install --no-cache-dir \
|
python3 -m pip install --no-cache-dir \
|
||||||
torch \
|
torch==2.1.2 \
|
||||||
torchvision \
|
torchvision==0.16.2 \
|
||||||
torchaudio \
|
torchaudio==2.1.2 \
|
||||||
invisible_watermark && \
|
invisible_watermark && \
|
||||||
python3 -m pip install --no-cache-dir \
|
python3 -m pip install --no-cache-dir \
|
||||||
accelerate \
|
accelerate \
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
<!---
|
<!---
|
||||||
Copyright 2024- The HuggingFace Team. All rights reserved.
|
Copyright 2023- The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -52,16 +52,12 @@
|
|||||||
title: Image-to-image
|
title: Image-to-image
|
||||||
- local: using-diffusers/inpaint
|
- local: using-diffusers/inpaint
|
||||||
title: Inpainting
|
title: Inpainting
|
||||||
- local: using-diffusers/text-img2vid
|
|
||||||
title: Text or image-to-video
|
|
||||||
- local: using-diffusers/depth2img
|
- local: using-diffusers/depth2img
|
||||||
title: Depth-to-image
|
title: Depth-to-image
|
||||||
title: Tasks
|
title: Tasks
|
||||||
- sections:
|
- sections:
|
||||||
- local: using-diffusers/textual_inversion_inference
|
- local: using-diffusers/textual_inversion_inference
|
||||||
title: Textual inversion
|
title: Textual inversion
|
||||||
- local: using-diffusers/ip_adapter
|
|
||||||
title: IP-Adapter
|
|
||||||
- local: training/distributed_inference
|
- local: training/distributed_inference
|
||||||
title: Distributed inference with multiple GPUs
|
title: Distributed inference with multiple GPUs
|
||||||
- local: using-diffusers/reusing_seeds
|
- local: using-diffusers/reusing_seeds
|
||||||
@@ -325,8 +321,6 @@
|
|||||||
title: Text-to-image
|
title: Text-to-image
|
||||||
- local: api/pipelines/stable_diffusion/img2img
|
- local: api/pipelines/stable_diffusion/img2img
|
||||||
title: Image-to-image
|
title: Image-to-image
|
||||||
- local: api/pipelines/stable_diffusion/svd
|
|
||||||
title: Image-to-video
|
|
||||||
- local: api/pipelines/stable_diffusion/inpaint
|
- local: api/pipelines/stable_diffusion/inpaint
|
||||||
title: Inpainting
|
title: Inpainting
|
||||||
- local: api/pipelines/stable_diffusion/depth2img
|
- local: api/pipelines/stable_diffusion/depth2img
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
@@ -20,24 +20,6 @@ An attention processor is a class for applying different types of attention mech
|
|||||||
## AttnProcessor2_0
|
## AttnProcessor2_0
|
||||||
[[autodoc]] models.attention_processor.AttnProcessor2_0
|
[[autodoc]] models.attention_processor.AttnProcessor2_0
|
||||||
|
|
||||||
## AttnAddedKVProcessor
|
|
||||||
[[autodoc]] models.attention_processor.AttnAddedKVProcessor
|
|
||||||
|
|
||||||
## AttnAddedKVProcessor2_0
|
|
||||||
[[autodoc]] models.attention_processor.AttnAddedKVProcessor2_0
|
|
||||||
|
|
||||||
## CrossFrameAttnProcessor
|
|
||||||
[[autodoc]] pipelines.text_to_video_synthesis.pipeline_text_to_video_zero.CrossFrameAttnProcessor
|
|
||||||
|
|
||||||
## CustomDiffusionAttnProcessor
|
|
||||||
[[autodoc]] models.attention_processor.CustomDiffusionAttnProcessor
|
|
||||||
|
|
||||||
## CustomDiffusionAttnProcessor2_0
|
|
||||||
[[autodoc]] models.attention_processor.CustomDiffusionAttnProcessor2_0
|
|
||||||
|
|
||||||
## CustomDiffusionXFormersAttnProcessor
|
|
||||||
[[autodoc]] models.attention_processor.CustomDiffusionXFormersAttnProcessor
|
|
||||||
|
|
||||||
## FusedAttnProcessor2_0
|
## FusedAttnProcessor2_0
|
||||||
[[autodoc]] models.attention_processor.FusedAttnProcessor2_0
|
[[autodoc]] models.attention_processor.FusedAttnProcessor2_0
|
||||||
|
|
||||||
@@ -47,17 +29,32 @@ An attention processor is a class for applying different types of attention mech
|
|||||||
## LoRAAttnProcessor2_0
|
## LoRAAttnProcessor2_0
|
||||||
[[autodoc]] models.attention_processor.LoRAAttnProcessor2_0
|
[[autodoc]] models.attention_processor.LoRAAttnProcessor2_0
|
||||||
|
|
||||||
|
## CustomDiffusionAttnProcessor
|
||||||
|
[[autodoc]] models.attention_processor.CustomDiffusionAttnProcessor
|
||||||
|
|
||||||
|
## CustomDiffusionAttnProcessor2_0
|
||||||
|
[[autodoc]] models.attention_processor.CustomDiffusionAttnProcessor2_0
|
||||||
|
|
||||||
|
## AttnAddedKVProcessor
|
||||||
|
[[autodoc]] models.attention_processor.AttnAddedKVProcessor
|
||||||
|
|
||||||
|
## AttnAddedKVProcessor2_0
|
||||||
|
[[autodoc]] models.attention_processor.AttnAddedKVProcessor2_0
|
||||||
|
|
||||||
## LoRAAttnAddedKVProcessor
|
## LoRAAttnAddedKVProcessor
|
||||||
[[autodoc]] models.attention_processor.LoRAAttnAddedKVProcessor
|
[[autodoc]] models.attention_processor.LoRAAttnAddedKVProcessor
|
||||||
|
|
||||||
|
## XFormersAttnProcessor
|
||||||
|
[[autodoc]] models.attention_processor.XFormersAttnProcessor
|
||||||
|
|
||||||
## LoRAXFormersAttnProcessor
|
## LoRAXFormersAttnProcessor
|
||||||
[[autodoc]] models.attention_processor.LoRAXFormersAttnProcessor
|
[[autodoc]] models.attention_processor.LoRAXFormersAttnProcessor
|
||||||
|
|
||||||
|
## CustomDiffusionXFormersAttnProcessor
|
||||||
|
[[autodoc]] models.attention_processor.CustomDiffusionXFormersAttnProcessor
|
||||||
|
|
||||||
## SlicedAttnProcessor
|
## SlicedAttnProcessor
|
||||||
[[autodoc]] models.attention_processor.SlicedAttnProcessor
|
[[autodoc]] models.attention_processor.SlicedAttnProcessor
|
||||||
|
|
||||||
## SlicedAttnAddedKVProcessor
|
## SlicedAttnAddedKVProcessor
|
||||||
[[autodoc]] models.attention_processor.SlicedAttnAddedKVProcessor
|
[[autodoc]] models.attention_processor.SlicedAttnAddedKVProcessor
|
||||||
|
|
||||||
## XFormersAttnProcessor
|
|
||||||
[[autodoc]] models.attention_processor.XFormersAttnProcessor
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
@@ -12,11 +12,11 @@ specific language governing permissions and limitations under the License.
|
|||||||
|
|
||||||
# IP-Adapter
|
# IP-Adapter
|
||||||
|
|
||||||
[IP-Adapter](https://hf.co/papers/2308.06721) is a lightweight adapter that enables prompting a diffusion model with an image. This method decouples the cross-attention layers of the image and text features. The image features are generated from an image encoder.
|
[IP-Adapter](https://hf.co/papers/2308.06721) is a lightweight adapter that enables prompting a diffusion model with an image. This method decouples the cross-attention layers of the image and text features. The image features are generated from an image encoder. Files generated from IP-Adapter are only ~100MBs.
|
||||||
|
|
||||||
<Tip>
|
<Tip>
|
||||||
|
|
||||||
Learn how to load an IP-Adapter checkpoint and image in the IP-Adapter [loading](../../using-diffusers/loading_adapters#ip-adapter) guide, and you can see how to use it in the [usage](../../using-diffusers/ip_adapter) guide.
|
Learn how to load an IP-Adapter checkpoint and image in the [IP-Adapter](../../using-diffusers/loading_adapters#ip-adapter) loading guide.
|
||||||
|
|
||||||
</Tip>
|
</Tip>
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
@@ -408,91 +408,6 @@ Make sure to check out the Schedulers [guide](../../using-diffusers/schedulers)
|
|||||||
|
|
||||||
</Tip>
|
</Tip>
|
||||||
|
|
||||||
## Using AnimateLCM
|
|
||||||
|
|
||||||
[AnimateLCM](https://animatelcm.github.io/) is a motion module checkpoint and an [LCM LoRA](https://huggingface.co/docs/diffusers/using-diffusers/inference_with_lcm_lora) that have been created using a consistency learning strategy that decouples the distillation of the image generation priors and the motion generation priors.
|
|
||||||
|
|
||||||
```python
|
|
||||||
import torch
|
|
||||||
from diffusers import AnimateDiffPipeline, LCMScheduler, MotionAdapter
|
|
||||||
from diffusers.utils import export_to_gif
|
|
||||||
|
|
||||||
adapter = MotionAdapter.from_pretrained("wangfuyun/AnimateLCM")
|
|
||||||
pipe = AnimateDiffPipeline.from_pretrained("emilianJR/epiCRealism", motion_adapter=adapter)
|
|
||||||
pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config, beta_schedule="linear")
|
|
||||||
|
|
||||||
pipe.load_lora_weights("wangfuyun/AnimateLCM", weight_name="sd15_lora_beta.safetensors", adapter_name="lcm-lora")
|
|
||||||
|
|
||||||
pipe.enable_vae_slicing()
|
|
||||||
pipe.enable_model_cpu_offload()
|
|
||||||
|
|
||||||
output = pipe(
|
|
||||||
prompt="A space rocket with trails of smoke behind it launching into space from the desert, 4k, high resolution",
|
|
||||||
negative_prompt="bad quality, worse quality, low resolution",
|
|
||||||
num_frames=16,
|
|
||||||
guidance_scale=1.5,
|
|
||||||
num_inference_steps=6,
|
|
||||||
generator=torch.Generator("cpu").manual_seed(0),
|
|
||||||
)
|
|
||||||
frames = output.frames[0]
|
|
||||||
export_to_gif(frames, "animatelcm.gif")
|
|
||||||
```
|
|
||||||
|
|
||||||
<table>
|
|
||||||
<tr>
|
|
||||||
<td><center>
|
|
||||||
A space rocket, 4K.
|
|
||||||
<br>
|
|
||||||
<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/animatelcm-output.gif"
|
|
||||||
alt="masterpiece, bestquality, sunset"
|
|
||||||
style="width: 300px;" />
|
|
||||||
</center></td>
|
|
||||||
</tr>
|
|
||||||
</table>
|
|
||||||
|
|
||||||
AnimateLCM is also compatible with existing [Motion LoRAs](https://huggingface.co/collections/dn6/animatediff-motion-loras-654cb8ad732b9e3cf4d3c17e).
|
|
||||||
|
|
||||||
```python
|
|
||||||
import torch
|
|
||||||
from diffusers import AnimateDiffPipeline, LCMScheduler, MotionAdapter
|
|
||||||
from diffusers.utils import export_to_gif
|
|
||||||
|
|
||||||
adapter = MotionAdapter.from_pretrained("wangfuyun/AnimateLCM")
|
|
||||||
pipe = AnimateDiffPipeline.from_pretrained("emilianJR/epiCRealism", motion_adapter=adapter)
|
|
||||||
pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config, beta_schedule="linear")
|
|
||||||
|
|
||||||
pipe.load_lora_weights("wangfuyun/AnimateLCM", weight_name="sd15_lora_beta.safetensors", adapter_name="lcm-lora")
|
|
||||||
pipe.load_lora_weights("guoyww/animatediff-motion-lora-tilt-up", adapter_name="tilt-up")
|
|
||||||
|
|
||||||
pipe.set_adapters(["lcm-lora", "tilt-up"], [1.0, 0.8])
|
|
||||||
pipe.enable_vae_slicing()
|
|
||||||
pipe.enable_model_cpu_offload()
|
|
||||||
|
|
||||||
output = pipe(
|
|
||||||
prompt="A space rocket with trails of smoke behind it launching into space from the desert, 4k, high resolution",
|
|
||||||
negative_prompt="bad quality, worse quality, low resolution",
|
|
||||||
num_frames=16,
|
|
||||||
guidance_scale=1.5,
|
|
||||||
num_inference_steps=6,
|
|
||||||
generator=torch.Generator("cpu").manual_seed(0),
|
|
||||||
)
|
|
||||||
frames = output.frames[0]
|
|
||||||
export_to_gif(frames, "animatelcm-motion-lora.gif")
|
|
||||||
```
|
|
||||||
|
|
||||||
<table>
|
|
||||||
<tr>
|
|
||||||
<td><center>
|
|
||||||
A space rocket, 4K.
|
|
||||||
<br>
|
|
||||||
<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/animatelcm-motion-lora.gif"
|
|
||||||
alt="masterpiece, bestquality, sunset"
|
|
||||||
style="width: 300px;" />
|
|
||||||
</center></td>
|
|
||||||
</tr>
|
|
||||||
</table>
|
|
||||||
|
|
||||||
|
|
||||||
## AnimateDiffPipeline
|
## AnimateDiffPipeline
|
||||||
|
|
||||||
[[autodoc]] AnimateDiffPipeline
|
[[autodoc]] AnimateDiffPipeline
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
@@ -31,7 +31,7 @@ Sample output with I2VGenXL:
|
|||||||
<table>
|
<table>
|
||||||
<tr>
|
<tr>
|
||||||
<td><center>
|
<td><center>
|
||||||
library.
|
masterpiece, bestquality, sunset.
|
||||||
<br>
|
<br>
|
||||||
<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/i2vgen-xl-example.gif"
|
<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/i2vgen-xl-example.gif"
|
||||||
alt="library"
|
alt="library"
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
@@ -70,7 +70,7 @@ Here are some sample outputs:
|
|||||||
<table>
|
<table>
|
||||||
<tr>
|
<tr>
|
||||||
<td><center>
|
<td><center>
|
||||||
cat in a field.
|
masterpiece, bestquality, sunset.
|
||||||
<br>
|
<br>
|
||||||
<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/pia-default-output.gif"
|
<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/pia-default-output.gif"
|
||||||
alt="cat in a field"
|
alt="cat in a field"
|
||||||
@@ -119,7 +119,7 @@ image = load_image(
|
|||||||
"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/pix2pix/cat_6.png?download=true"
|
"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/pix2pix/cat_6.png?download=true"
|
||||||
)
|
)
|
||||||
image = image.resize((512, 512))
|
image = image.resize((512, 512))
|
||||||
prompt = "cat in a field"
|
prompt = "cat in a hat"
|
||||||
negative_prompt = "wrong white balance, dark, sketches,worst quality,low quality"
|
negative_prompt = "wrong white balance, dark, sketches,worst quality,low quality"
|
||||||
|
|
||||||
generator = torch.Generator("cpu").manual_seed(0)
|
generator = torch.Generator("cpu").manual_seed(0)
|
||||||
@@ -132,7 +132,7 @@ export_to_gif(frames, "pia-freeinit-animation.gif")
|
|||||||
<table>
|
<table>
|
||||||
<tr>
|
<tr>
|
||||||
<td><center>
|
<td><center>
|
||||||
cat in a field.
|
masterpiece, bestquality, sunset.
|
||||||
<br>
|
<br>
|
||||||
<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/pia-freeinit-output-cat.gif"
|
<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/pia-freeinit-output-cat.gif"
|
||||||
alt="cat in a field"
|
alt="cat in a field"
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The GLIGEN Authors and The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The GLIGEN Authors and The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The Intel Labs Team Authors and HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The Intel Labs Team Authors and HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,43 +0,0 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
|
||||||
the License. You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
|
||||||
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
|
||||||
specific language governing permissions and limitations under the License.
|
|
||||||
-->
|
|
||||||
|
|
||||||
# Stable Video Diffusion
|
|
||||||
|
|
||||||
Stable Video Diffusion was proposed in [Stable Video Diffusion: Scaling Latent Video Diffusion Models to Large Datasets](https://hf.co/papers/2311.15127) by Andreas Blattmann, Tim Dockhorn, Sumith Kulal, Daniel Mendelevitch, Maciej Kilian, Dominik Lorenz, Yam Levi, Zion English, Vikram Voleti, Adam Letts, Varun Jampani, Robin Rombach.
|
|
||||||
|
|
||||||
The abstract from the paper is:
|
|
||||||
|
|
||||||
*We present Stable Video Diffusion - a latent video diffusion model for high-resolution, state-of-the-art text-to-video and image-to-video generation. Recently, latent diffusion models trained for 2D image synthesis have been turned into generative video models by inserting temporal layers and finetuning them on small, high-quality video datasets. However, training methods in the literature vary widely, and the field has yet to agree on a unified strategy for curating video data. In this paper, we identify and evaluate three different stages for successful training of video LDMs: text-to-image pretraining, video pretraining, and high-quality video finetuning. Furthermore, we demonstrate the necessity of a well-curated pretraining dataset for generating high-quality videos and present a systematic curation process to train a strong base model, including captioning and filtering strategies. We then explore the impact of finetuning our base model on high-quality data and train a text-to-video model that is competitive with closed-source video generation. We also show that our base model provides a powerful motion representation for downstream tasks such as image-to-video generation and adaptability to camera motion-specific LoRA modules. Finally, we demonstrate that our model provides a strong multi-view 3D-prior and can serve as a base to finetune a multi-view diffusion model that jointly generates multiple views of objects in a feedforward fashion, outperforming image-based methods at a fraction of their compute budget. We release code and model weights at this https URL.*
|
|
||||||
|
|
||||||
<Tip>
|
|
||||||
|
|
||||||
To learn how to use Stable Video Diffusion, take a look at the [Stable Video Diffusion](../../../using-diffusers/svd) guide.
|
|
||||||
|
|
||||||
<br>
|
|
||||||
|
|
||||||
Check out the [Stability AI](https://huggingface.co/stabilityai) Hub organization for the [base](https://huggingface.co/stabilityai/stable-video-diffusion-img2vid) and [extended frame](https://huggingface.co/stabilityai/stable-video-diffusion-img2vid-xt) checkpoints!
|
|
||||||
|
|
||||||
</Tip>
|
|
||||||
|
|
||||||
## Tips
|
|
||||||
|
|
||||||
Video generation is memory-intensive and one way to reduce your memory usage is to set `enable_forward_chunking` on the pipeline's UNet so you don't run the entire feedforward layer at once. Breaking it up into chunks in a loop is more efficient.
|
|
||||||
|
|
||||||
Check out the [Text or image-to-video](text-img2vid) guide for more details about how certain parameters can affect video generation and how to optimize inference by reducing memory usage.
|
|
||||||
|
|
||||||
## StableVideoDiffusionPipeline
|
|
||||||
|
|
||||||
[[autodoc]] StableVideoDiffusionPipeline
|
|
||||||
|
|
||||||
## StableVideoDiffusionPipelineOutput
|
|
||||||
|
|
||||||
[[autodoc]] pipelines.stable_video_diffusion.StableVideoDiffusionPipelineOutput
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
@@ -41,7 +41,7 @@ pipe = DiffusionPipeline.from_pretrained("damo-vilab/text-to-video-ms-1.7b", tor
|
|||||||
pipe = pipe.to("cuda")
|
pipe = pipe.to("cuda")
|
||||||
|
|
||||||
prompt = "Spiderman is surfing"
|
prompt = "Spiderman is surfing"
|
||||||
video_frames = pipe(prompt).frames[0]
|
video_frames = pipe(prompt).frames
|
||||||
video_path = export_to_video(video_frames)
|
video_path = export_to_video(video_frames)
|
||||||
video_path
|
video_path
|
||||||
```
|
```
|
||||||
@@ -64,7 +64,7 @@ pipe.enable_model_cpu_offload()
|
|||||||
pipe.enable_vae_slicing()
|
pipe.enable_vae_slicing()
|
||||||
|
|
||||||
prompt = "Darth Vader surfing a wave"
|
prompt = "Darth Vader surfing a wave"
|
||||||
video_frames = pipe(prompt, num_frames=64).frames[0]
|
video_frames = pipe(prompt, num_frames=64).frames
|
||||||
video_path = export_to_video(video_frames)
|
video_path = export_to_video(video_frames)
|
||||||
video_path
|
video_path
|
||||||
```
|
```
|
||||||
@@ -83,7 +83,7 @@ pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
|
|||||||
pipe.enable_model_cpu_offload()
|
pipe.enable_model_cpu_offload()
|
||||||
|
|
||||||
prompt = "Spiderman is surfing"
|
prompt = "Spiderman is surfing"
|
||||||
video_frames = pipe(prompt, num_inference_steps=25).frames[0]
|
video_frames = pipe(prompt, num_inference_steps=25).frames
|
||||||
video_path = export_to_video(video_frames)
|
video_path = export_to_video(video_frames)
|
||||||
video_path
|
video_path
|
||||||
```
|
```
|
||||||
@@ -130,7 +130,7 @@ pipe.unet.enable_forward_chunking(chunk_size=1, dim=1)
|
|||||||
pipe.enable_vae_slicing()
|
pipe.enable_vae_slicing()
|
||||||
|
|
||||||
prompt = "Darth Vader surfing a wave"
|
prompt = "Darth Vader surfing a wave"
|
||||||
video_frames = pipe(prompt, num_frames=24).frames[0]
|
video_frames = pipe(prompt, num_frames=24).frames
|
||||||
video_path = export_to_video(video_frames)
|
video_path = export_to_video(video_frames)
|
||||||
video_path
|
video_path
|
||||||
```
|
```
|
||||||
@@ -148,7 +148,7 @@ pipe.enable_vae_slicing()
|
|||||||
|
|
||||||
video = [Image.fromarray(frame).resize((1024, 576)) for frame in video_frames]
|
video = [Image.fromarray(frame).resize((1024, 576)) for frame in video_frames]
|
||||||
|
|
||||||
video_frames = pipe(prompt, video=video, strength=0.6).frames[0]
|
video_frames = pipe(prompt, video=video, strength=0.6).frames
|
||||||
video_path = export_to_video(video_frames)
|
video_path = export_to_video(video_frames)
|
||||||
video_path
|
video_path
|
||||||
```
|
```
|
||||||
@@ -167,12 +167,6 @@ Here are some sample outputs:
|
|||||||
</tr>
|
</tr>
|
||||||
</table>
|
</table>
|
||||||
|
|
||||||
## Tips
|
|
||||||
|
|
||||||
Video generation is memory-intensive and one way to reduce your memory usage is to set `enable_forward_chunking` on the pipeline's UNet so you don't run the entire feedforward layer at once. Breaking it up into chunks in a loop is more efficient.
|
|
||||||
|
|
||||||
Check out the [Text or image-to-video](text-img2vid) guide for more details about how certain parameters can affect video generation and how to optimize inference by reducing memory usage.
|
|
||||||
|
|
||||||
<Tip>
|
<Tip>
|
||||||
|
|
||||||
Make sure to check out the Schedulers [guide](../../using-diffusers/schedulers) to learn how to explore the tradeoff between scheduler speed and quality, and see the [reuse components across pipelines](../../using-diffusers/loading#reuse-components-across-pipelines) section to learn how to efficiently load the same components into multiple pipelines.
|
Make sure to check out the Schedulers [guide](../../using-diffusers/schedulers) to learn how to explore the tradeoff between scheduler speed and quality, and see the [reuse components across pipelines](../../using-diffusers/loading#reuse-components-across-pipelines) section to learn how to efficiently load the same components into multiple pipelines.
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<!--Copyright 2024 The HuggingFace Team. All rights reserved.
|
<!--Copyright 2023 The HuggingFace Team. All rights reserved.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||||
the License. You may obtain a copy of the License at
|
the License. You may obtain a copy of the License at
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user