mirror of
https://github.com/huggingface/diffusers.git
synced 2025-12-15 17:04:52 +08:00
Compare commits
4 Commits
modular-do
...
modular-lo
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e30dc5176c | ||
|
|
ca4a16f94c | ||
|
|
30a72ee140 | ||
|
|
58dfcf9e92 |
@@ -51,10 +51,10 @@ t2i_pipeline = t2i_blocks.init_pipeline(modular_repo_id, components_manager=comp
|
|||||||
</hfoption>
|
</hfoption>
|
||||||
</hfoptions>
|
</hfoptions>
|
||||||
|
|
||||||
Components are only loaded and registered when using [`~ModularPipeline.load_components`] or [`~ModularPipeline.load_default_components`]. The example below uses [`~ModularPipeline.load_default_components`] to create a second pipeline that reuses all the components from the first one, and assigns it to a different collection
|
Components are only loaded and registered when using [`~ModularPipeline.load_components`] or [`~ModularPipeline.load_components`]. The example below uses [`~ModularPipeline.load_components`] to create a second pipeline that reuses all the components from the first one, and assigns it to a different collection
|
||||||
|
|
||||||
```py
|
```py
|
||||||
pipe.load_default_components()
|
pipe.load_components()
|
||||||
pipe2 = ModularPipeline.from_pretrained("YiYiXu/modular-demo-auto", components_manager=comp, collection="test2")
|
pipe2 = ModularPipeline.from_pretrained("YiYiXu/modular-demo-auto", components_manager=comp, collection="test2")
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -187,4 +187,4 @@ comp.enable_auto_cpu_offload(device="cuda")
|
|||||||
|
|
||||||
All models begin on the CPU and [`ComponentsManager`] moves them to the appropriate device right before they're needed, and moves other models back to the CPU when GPU memory is low.
|
All models begin on the CPU and [`ComponentsManager`] moves them to the appropriate device right before they're needed, and moves other models back to the CPU when GPU memory is low.
|
||||||
|
|
||||||
You can set your own rules for which models to offload first.
|
You can set your own rules for which models to offload first.
|
||||||
|
|||||||
@@ -75,13 +75,13 @@ Guiders that are already saved on the Hub with a `modular_model_index.json` file
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
The guider is only created after calling [`~ModularPipeline.load_default_components`] based on the loading specification in `modular_model_index.json`.
|
The guider is only created after calling [`~ModularPipeline.load_components`] based on the loading specification in `modular_model_index.json`.
|
||||||
|
|
||||||
```py
|
```py
|
||||||
t2i_pipeline = t2i_blocks.init_pipeline("YiYiXu/modular-doc-guider")
|
t2i_pipeline = t2i_blocks.init_pipeline("YiYiXu/modular-doc-guider")
|
||||||
# not created during init
|
# not created during init
|
||||||
assert t2i_pipeline.guider is None
|
assert t2i_pipeline.guider is None
|
||||||
t2i_pipeline.load_default_components()
|
t2i_pipeline.load_components()
|
||||||
# loaded as PAG guider
|
# loaded as PAG guider
|
||||||
t2i_pipeline.guider
|
t2i_pipeline.guider
|
||||||
```
|
```
|
||||||
@@ -172,4 +172,4 @@ t2i_pipeline.push_to_hub("YiYiXu/modular-doc-guider")
|
|||||||
```
|
```
|
||||||
|
|
||||||
</hfoption>
|
</hfoption>
|
||||||
</hfoptions>
|
</hfoptions>
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ blocks = SequentialPipelineBlocks.from_blocks_dict(TEXT2IMAGE_BLOCKS)
|
|||||||
modular_repo_id = "YiYiXu/modular-loader-t2i-0704"
|
modular_repo_id = "YiYiXu/modular-loader-t2i-0704"
|
||||||
pipeline = blocks.init_pipeline(modular_repo_id)
|
pipeline = blocks.init_pipeline(modular_repo_id)
|
||||||
|
|
||||||
pipeline.load_default_components(torch_dtype=torch.float16)
|
pipeline.load_components(torch_dtype=torch.float16)
|
||||||
pipeline.to("cuda")
|
pipeline.to("cuda")
|
||||||
|
|
||||||
image = pipeline(prompt="Astronaut in a jungle, cold color palette, muted colors, detailed, 8k", output="images")[0]
|
image = pipeline(prompt="Astronaut in a jungle, cold color palette, muted colors, detailed, 8k", output="images")[0]
|
||||||
@@ -49,7 +49,7 @@ blocks = SequentialPipelineBlocks.from_blocks_dict(IMAGE2IMAGE_BLOCKS)
|
|||||||
modular_repo_id = "YiYiXu/modular-loader-t2i-0704"
|
modular_repo_id = "YiYiXu/modular-loader-t2i-0704"
|
||||||
pipeline = blocks.init_pipeline(modular_repo_id)
|
pipeline = blocks.init_pipeline(modular_repo_id)
|
||||||
|
|
||||||
pipeline.load_default_components(torch_dtype=torch.float16)
|
pipeline.load_components(torch_dtype=torch.float16)
|
||||||
pipeline.to("cuda")
|
pipeline.to("cuda")
|
||||||
|
|
||||||
url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl-text2img.png"
|
url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl-text2img.png"
|
||||||
@@ -73,7 +73,7 @@ blocks = SequentialPipelineBlocks.from_blocks_dict(INPAINT_BLOCKS)
|
|||||||
modular_repo_id = "YiYiXu/modular-loader-t2i-0704"
|
modular_repo_id = "YiYiXu/modular-loader-t2i-0704"
|
||||||
pipeline = blocks.init_pipeline(modular_repo_id)
|
pipeline = blocks.init_pipeline(modular_repo_id)
|
||||||
|
|
||||||
pipeline.load_default_components(torch_dtype=torch.float16)
|
pipeline.load_components(torch_dtype=torch.float16)
|
||||||
pipeline.to("cuda")
|
pipeline.to("cuda")
|
||||||
|
|
||||||
img_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl-text2img.png"
|
img_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl-text2img.png"
|
||||||
@@ -176,15 +176,15 @@ diffdiff_pipeline = ModularPipeline.from_pretrained(modular_repo_id, trust_remot
|
|||||||
|
|
||||||
## Loading components
|
## Loading components
|
||||||
|
|
||||||
A [`ModularPipeline`] doesn't automatically instantiate with components. It only loads the configuration and component specifications. You can load all components with [`~ModularPipeline.load_default_components`] or only load specific components with [`~ModularPipeline.load_components`].
|
A [`ModularPipeline`] doesn't automatically instantiate with components. It only loads the configuration and component specifications. You can load all components with [`~ModularPipeline.load_components`] or only load specific components with [`~ModularPipeline.load_components`].
|
||||||
|
|
||||||
<hfoptions id="load">
|
<hfoptions id="load">
|
||||||
<hfoption id="load_default_components">
|
<hfoption id="load_components">
|
||||||
|
|
||||||
```py
|
```py
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
t2i_pipeline.load_default_components(torch_dtype=torch.float16)
|
t2i_pipeline.load_components(torch_dtype=torch.float16)
|
||||||
t2i_pipeline.to("cuda")
|
t2i_pipeline.to("cuda")
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -355,4 +355,4 @@ The [config.json](https://huggingface.co/YiYiXu/modular-diffdiff-0704/blob/main/
|
|||||||
"ModularPipelineBlocks": "block.DiffDiffBlocks"
|
"ModularPipelineBlocks": "block.DiffDiffBlocks"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -173,9 +173,9 @@ print(dd_blocks)
|
|||||||
|
|
||||||
## ModularPipeline
|
## ModularPipeline
|
||||||
|
|
||||||
Convert the [`SequentialPipelineBlocks`] into a [`ModularPipeline`] with the [`ModularPipeline.init_pipeline`] method. This initializes the expected components to load from a `modular_model_index.json` file. Explicitly load the components by calling [`ModularPipeline.load_default_components`].
|
Convert the [`SequentialPipelineBlocks`] into a [`ModularPipeline`] with the [`ModularPipeline.init_pipeline`] method. This initializes the expected components to load from a `modular_model_index.json` file. Explicitly load the components by calling [`ModularPipeline.load_components`].
|
||||||
|
|
||||||
It is a good idea to initialize the [`ComponentManager`] with the pipeline to help manage the different components. Once you call [`~ModularPipeline.load_default_components`], the components are registered to the [`ComponentManager`] and can be shared between workflows. The example below uses the `collection` argument to assign the components a `"diffdiff"` label for better organization.
|
It is a good idea to initialize the [`ComponentManager`] with the pipeline to help manage the different components. Once you call [`~ModularPipeline.load_components`], the components are registered to the [`ComponentManager`] and can be shared between workflows. The example below uses the `collection` argument to assign the components a `"diffdiff"` label for better organization.
|
||||||
|
|
||||||
```py
|
```py
|
||||||
from diffusers.modular_pipelines import ComponentsManager
|
from diffusers.modular_pipelines import ComponentsManager
|
||||||
@@ -209,11 +209,11 @@ Use the [`sub_blocks.insert`] method to insert it into the [`ModularPipeline`].
|
|||||||
dd_blocks.sub_blocks.insert("ip_adapter", ip_adapter_block, 0)
|
dd_blocks.sub_blocks.insert("ip_adapter", ip_adapter_block, 0)
|
||||||
```
|
```
|
||||||
|
|
||||||
Call [`~ModularPipeline.init_pipeline`] to initialize a [`ModularPipeline`] and use [`~ModularPipeline.load_default_components`] to load the model components. Load and set the IP-Adapter to run the pipeline.
|
Call [`~ModularPipeline.init_pipeline`] to initialize a [`ModularPipeline`] and use [`~ModularPipeline.load_components`] to load the model components. Load and set the IP-Adapter to run the pipeline.
|
||||||
|
|
||||||
```py
|
```py
|
||||||
dd_pipeline = dd_blocks.init_pipeline("YiYiXu/modular-demo-auto", collection="diffdiff")
|
dd_pipeline = dd_blocks.init_pipeline("YiYiXu/modular-demo-auto", collection="diffdiff")
|
||||||
dd_pipeline.load_default_components(torch_dtype=torch.float16)
|
dd_pipeline.load_components(torch_dtype=torch.float16)
|
||||||
dd_pipeline.loader.load_ip_adapter("h94/IP-Adapter", subfolder="sdxl_models", weight_name="ip-adapter_sdxl.bin")
|
dd_pipeline.loader.load_ip_adapter("h94/IP-Adapter", subfolder="sdxl_models", weight_name="ip-adapter_sdxl.bin")
|
||||||
dd_pipeline.loader.set_ip_adapter_scale(0.6)
|
dd_pipeline.loader.set_ip_adapter_scale(0.6)
|
||||||
dd_pipeline = dd_pipeline.to(device)
|
dd_pipeline = dd_pipeline.to(device)
|
||||||
@@ -260,14 +260,14 @@ class SDXLDiffDiffControlNetDenoiseStep(StableDiffusionXLDenoiseLoopWrapper):
|
|||||||
controlnet_denoise_block = SDXLDiffDiffControlNetDenoiseStep()
|
controlnet_denoise_block = SDXLDiffDiffControlNetDenoiseStep()
|
||||||
```
|
```
|
||||||
|
|
||||||
Insert the `controlnet_input` block and replace the `denoise` block with the new `controlnet_denoise_block`. Initialize a [`ModularPipeline`] and [`~ModularPipeline.load_default_components`] into it.
|
Insert the `controlnet_input` block and replace the `denoise` block with the new `controlnet_denoise_block`. Initialize a [`ModularPipeline`] and [`~ModularPipeline.load_components`] into it.
|
||||||
|
|
||||||
```py
|
```py
|
||||||
dd_blocks.sub_blocks.insert("controlnet_input", control_input_block, 7)
|
dd_blocks.sub_blocks.insert("controlnet_input", control_input_block, 7)
|
||||||
dd_blocks.sub_blocks["denoise"] = controlnet_denoise_block
|
dd_blocks.sub_blocks["denoise"] = controlnet_denoise_block
|
||||||
|
|
||||||
dd_pipeline = dd_blocks.init_pipeline("YiYiXu/modular-demo-auto", collection="diffdiff")
|
dd_pipeline = dd_blocks.init_pipeline("YiYiXu/modular-demo-auto", collection="diffdiff")
|
||||||
dd_pipeline.load_default_components(torch_dtype=torch.float16)
|
dd_pipeline.load_components(torch_dtype=torch.float16)
|
||||||
dd_pipeline = dd_pipeline.to(device)
|
dd_pipeline = dd_pipeline.to(device)
|
||||||
|
|
||||||
control_image = load_image("https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/diffdiff_tomato_canny.jpeg")
|
control_image = load_image("https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/diffdiff_tomato_canny.jpeg")
|
||||||
@@ -320,7 +320,7 @@ Call [`SequentialPipelineBlocks.from_blocks_dict`] to create a [`SequentialPipel
|
|||||||
```py
|
```py
|
||||||
dd_auto_blocks = SequentialPipelineBlocks.from_blocks_dict(DIFFDIFF_AUTO_BLOCKS)
|
dd_auto_blocks = SequentialPipelineBlocks.from_blocks_dict(DIFFDIFF_AUTO_BLOCKS)
|
||||||
dd_pipeline = dd_auto_blocks.init_pipeline("YiYiXu/modular-demo-auto", collection="diffdiff")
|
dd_pipeline = dd_auto_blocks.init_pipeline("YiYiXu/modular-demo-auto", collection="diffdiff")
|
||||||
dd_pipeline.load_default_components(torch_dtype=torch.float16)
|
dd_pipeline.load_components(torch_dtype=torch.float16)
|
||||||
```
|
```
|
||||||
|
|
||||||
## Share
|
## Share
|
||||||
@@ -340,5 +340,5 @@ from diffusers.modular_pipelines import ModularPipeline, ComponentsManager
|
|||||||
components = ComponentsManager()
|
components = ComponentsManager()
|
||||||
|
|
||||||
diffdiff_pipeline = ModularPipeline.from_pretrained("YiYiXu/modular-diffdiff-0704", trust_remote_code=True, components_manager=components, collection="diffdiff")
|
diffdiff_pipeline = ModularPipeline.from_pretrained("YiYiXu/modular-diffdiff-0704", trust_remote_code=True, components_manager=components, collection="diffdiff")
|
||||||
diffdiff_pipeline.load_default_components(torch_dtype=torch.float16)
|
diffdiff_pipeline.load_components(torch_dtype=torch.float16)
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -48,10 +48,10 @@ t2i_pipeline = t2i_blocks.init_pipeline(modular_repo_id, components_manager=comp
|
|||||||
</hfoption>
|
</hfoption>
|
||||||
</hfoptions>
|
</hfoptions>
|
||||||
|
|
||||||
组件仅在调用 [`~ModularPipeline.load_components`] 或 [`~ModularPipeline.load_default_components`] 时加载和注册。以下示例使用 [`~ModularPipeline.load_default_components`] 创建第二个管道,重用第一个管道的所有组件,并将其分配到不同的集合。
|
组件仅在调用 [`~ModularPipeline.load_components`] 或 [`~ModularPipeline.load_components`] 时加载和注册。以下示例使用 [`~ModularPipeline.load_components`] 创建第二个管道,重用第一个管道的所有组件,并将其分配到不同的集合。
|
||||||
|
|
||||||
```py
|
```py
|
||||||
pipe.load_default_components()
|
pipe.load_components()
|
||||||
pipe2 = ModularPipeline.from_pretrained("YiYiXu/modular-demo-auto", components_manager=comp, collection="test2")
|
pipe2 = ModularPipeline.from_pretrained("YiYiXu/modular-demo-auto", components_manager=comp, collection="test2")
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -185,4 +185,4 @@ comp.enable_auto_cpu_offload(device="cuda")
|
|||||||
|
|
||||||
所有模型开始时都在 CPU 上,[`ComponentsManager`] 在需要它们之前将它们移动到适当的设备,并在 GPU 内存不足时将其他模型移回 CPU。
|
所有模型开始时都在 CPU 上,[`ComponentsManager`] 在需要它们之前将它们移动到适当的设备,并在 GPU 内存不足时将其他模型移回 CPU。
|
||||||
|
|
||||||
您可以设置自己的规则来决定哪些模型要卸载。
|
您可以设置自己的规则来决定哪些模型要卸载。
|
||||||
|
|||||||
@@ -73,13 +73,13 @@ ComponentSpec(name='guider', type_hint=<class 'diffusers.guiders.perturbed_atten
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
引导器只有在调用 [`~ModularPipeline.load_default_components`] 之后才会创建,基于 `modular_model_index.json` 中的加载规范。
|
引导器只有在调用 [`~ModularPipeline.load_components`] 之后才会创建,基于 `modular_model_index.json` 中的加载规范。
|
||||||
|
|
||||||
```py
|
```py
|
||||||
t2i_pipeline = t2i_blocks.init_pipeline("YiYiXu/modular-doc-guider")
|
t2i_pipeline = t2i_blocks.init_pipeline("YiYiXu/modular-doc-guider")
|
||||||
# 在初始化时未创建
|
# 在初始化时未创建
|
||||||
assert t2i_pipeline.guider is None
|
assert t2i_pipeline.guider is None
|
||||||
t2i_pipeline.load_default_components()
|
t2i_pipeline.load_components()
|
||||||
# 加载为 PAG 引导器
|
# 加载为 PAG 引导器
|
||||||
t2i_pipeline.guider
|
t2i_pipeline.guider
|
||||||
```
|
```
|
||||||
@@ -170,4 +170,4 @@ t2i_pipeline.push_to_hub("YiYiXu/modular-doc-guider")
|
|||||||
```
|
```
|
||||||
|
|
||||||
</hfoption>
|
</hfoption>
|
||||||
</hfoptions>
|
</hfoptions>
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ blocks = SequentialPipelineBlocks.from_blocks_dict(TEXT2IMAGE_BLOCKS)
|
|||||||
modular_repo_id = "YiYiXu/modular-loader-t2i-0704"
|
modular_repo_id = "YiYiXu/modular-loader-t2i-0704"
|
||||||
pipeline = blocks.init_pipeline(modular_repo_id)
|
pipeline = blocks.init_pipeline(modular_repo_id)
|
||||||
|
|
||||||
pipeline.load_default_components(torch_dtype=torch.float16)
|
pipeline.load_components(torch_dtype=torch.float16)
|
||||||
pipeline.to("cuda")
|
pipeline.to("cuda")
|
||||||
|
|
||||||
image = pipeline(prompt="Astronaut in a jungle, cold color palette, muted colors, detailed, 8k", output="images")[0]
|
image = pipeline(prompt="Astronaut in a jungle, cold color palette, muted colors, detailed, 8k", output="images")[0]
|
||||||
@@ -48,7 +48,7 @@ blocks = SequentialPipelineBlocks.from_blocks_dict(IMAGE2IMAGE_BLOCKS)
|
|||||||
modular_repo_id = "YiYiXu/modular-loader-t2i-0704"
|
modular_repo_id = "YiYiXu/modular-loader-t2i-0704"
|
||||||
pipeline = blocks.init_pipeline(modular_repo_id)
|
pipeline = blocks.init_pipeline(modular_repo_id)
|
||||||
|
|
||||||
pipeline.load_default_components(torch_dtype=torch.float16)
|
pipeline.load_components(torch_dtype=torch.float16)
|
||||||
pipeline.to("cuda")
|
pipeline.to("cuda")
|
||||||
|
|
||||||
url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl-text2img.png"
|
url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl-text2img.png"
|
||||||
@@ -72,7 +72,7 @@ blocks = SequentialPipelineBlocks.from_blocks_dict(INPAINT_BLOCKS)
|
|||||||
modular_repo_id = "YiYiXu/modular-loader-t2i-0704"
|
modular_repo_id = "YiYiXu/modular-loader-t2i-0704"
|
||||||
pipeline = blocks.init_pipeline(modular_repo_id)
|
pipeline = blocks.init_pipeline(modular_repo_id)
|
||||||
|
|
||||||
pipeline.load_default_components(torch_dtype=torch.float16)
|
pipeline.load_components(torch_dtype=torch.float16)
|
||||||
pipeline.to("cuda")
|
pipeline.to("cuda")
|
||||||
|
|
||||||
img_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl-text2img.png"
|
img_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl-text2img.png"
|
||||||
@@ -176,15 +176,15 @@ diffdiff_pipeline = ModularPipeline.from_pretrained(modular_repo_id, trust_remot
|
|||||||
|
|
||||||
## 加载组件
|
## 加载组件
|
||||||
|
|
||||||
一个[`ModularPipeline`]不会自动实例化组件。它只加载配置和组件规范。您可以使用[`~ModularPipeline.load_default_components`]加载所有组件,或仅使用[`~ModularPipeline.load_components`]加载特定组件。
|
一个[`ModularPipeline`]不会自动实例化组件。它只加载配置和组件规范。您可以使用[`~ModularPipeline.load_components`]加载所有组件,或仅使用[`~ModularPipeline.load_components`]加载特定组件。
|
||||||
|
|
||||||
<hfoptions id="load">
|
<hfoptions id="load">
|
||||||
<hfoption id="load_default_components">
|
<hfoption id="load_components">
|
||||||
|
|
||||||
```py
|
```py
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
t2i_pipeline.load_default_components(torch_dtype=torch.float16)
|
t2i_pipeline.load_components(torch_dtype=torch.float16)
|
||||||
t2i_pipeline.to("cuda")
|
t2i_pipeline.to("cuda")
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -175,7 +175,7 @@ print(dd_blocks)
|
|||||||
将 [`SequentialPipelineBlocks`] 转换为 [`ModularPipeline`],使用 [`ModularPipeline.init_pipeline`] 方法。这会初始化从 `modular_model_index.json` 文件加载的预期组件。通过调用 [`ModularPipeline.load_defau
|
将 [`SequentialPipelineBlocks`] 转换为 [`ModularPipeline`],使用 [`ModularPipeline.init_pipeline`] 方法。这会初始化从 `modular_model_index.json` 文件加载的预期组件。通过调用 [`ModularPipeline.load_defau
|
||||||
lt_components`]。
|
lt_components`]。
|
||||||
|
|
||||||
初始化[`ComponentManager`]时传入pipeline是一个好主意,以帮助管理不同的组件。一旦调用[`~ModularPipeline.load_default_components`],组件就会被注册到[`ComponentManager`]中,并且可以在工作流之间共享。下面的例子使用`collection`参数为组件分配了一个`"diffdiff"`标签,以便更好地组织。
|
初始化[`ComponentManager`]时传入pipeline是一个好主意,以帮助管理不同的组件。一旦调用[`~ModularPipeline.load_components`],组件就会被注册到[`ComponentManager`]中,并且可以在工作流之间共享。下面的例子使用`collection`参数为组件分配了一个`"diffdiff"`标签,以便更好地组织。
|
||||||
|
|
||||||
```py
|
```py
|
||||||
from diffusers.modular_pipelines import ComponentsManager
|
from diffusers.modular_pipelines import ComponentsManager
|
||||||
@@ -209,11 +209,11 @@ ip_adapter_block = StableDiffusionXLAutoIPAdapterStep()
|
|||||||
dd_blocks.sub_blocks.insert("ip_adapter", ip_adapter_block, 0)
|
dd_blocks.sub_blocks.insert("ip_adapter", ip_adapter_block, 0)
|
||||||
```
|
```
|
||||||
|
|
||||||
调用[`~ModularPipeline.init_pipeline`]来初始化一个[`ModularPipeline`],并使用[`~ModularPipeline.load_default_components`]加载模型组件。加载并设置IP-Adapter以运行pipeline。
|
调用[`~ModularPipeline.init_pipeline`]来初始化一个[`ModularPipeline`],并使用[`~ModularPipeline.load_components`]加载模型组件。加载并设置IP-Adapter以运行pipeline。
|
||||||
|
|
||||||
```py
|
```py
|
||||||
dd_pipeline = dd_blocks.init_pipeline("YiYiXu/modular-demo-auto", collection="diffdiff")
|
dd_pipeline = dd_blocks.init_pipeline("YiYiXu/modular-demo-auto", collection="diffdiff")
|
||||||
dd_pipeline.load_default_components(torch_dtype=torch.float16)
|
dd_pipeline.load_components(torch_dtype=torch.float16)
|
||||||
dd_pipeline.loader.load_ip_adapter("h94/IP-Adapter", subfolder="sdxl_models", weight_name="ip-adapter_sdxl.bin")
|
dd_pipeline.loader.load_ip_adapter("h94/IP-Adapter", subfolder="sdxl_models", weight_name="ip-adapter_sdxl.bin")
|
||||||
dd_pipeline.loader.set_ip_adapter_scale(0.6)
|
dd_pipeline.loader.set_ip_adapter_scale(0.6)
|
||||||
dd_pipeline = dd_pipeline.to(device)
|
dd_pipeline = dd_pipeline.to(device)
|
||||||
@@ -261,14 +261,14 @@ class SDXLDiffDiffControlNetDenoiseStep(StableDiffusionXLDenoiseLoopWrapper):
|
|||||||
controlnet_denoise_block = SDXLDiffDiffControlNetDenoiseStep()
|
controlnet_denoise_block = SDXLDiffDiffControlNetDenoiseStep()
|
||||||
```
|
```
|
||||||
|
|
||||||
插入 `controlnet_input` 块并用新的 `controlnet_denoise_block` 替换 `denoise` 块。初始化一个 [`ModularPipeline`] 并将 [`~ModularPipeline.load_default_components`] 加载到其中。
|
插入 `controlnet_input` 块并用新的 `controlnet_denoise_block` 替换 `denoise` 块。初始化一个 [`ModularPipeline`] 并将 [`~ModularPipeline.load_components`] 加载到其中。
|
||||||
|
|
||||||
```py
|
```py
|
||||||
dd_blocks.sub_blocks.insert("controlnet_input", control_input_block, 7)
|
dd_blocks.sub_blocks.insert("controlnet_input", control_input_block, 7)
|
||||||
dd_blocks.sub_blocks["denoise"] = controlnet_denoise_block
|
dd_blocks.sub_blocks["denoise"] = controlnet_denoise_block
|
||||||
|
|
||||||
dd_pipeline = dd_blocks.init_pipeline("YiYiXu/modular-demo-auto", collection="diffdiff")
|
dd_pipeline = dd_blocks.init_pipeline("YiYiXu/modular-demo-auto", collection="diffdiff")
|
||||||
dd_pipeline.load_default_components(torch_dtype=torch.float16)
|
dd_pipeline.load_components(torch_dtype=torch.float16)
|
||||||
dd_pipeline = dd_pipeline.to(device)
|
dd_pipeline = dd_pipeline.to(device)
|
||||||
|
|
||||||
control_image = load_image("https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/diffdiff_tomato_canny.jpeg")
|
control_image = load_image("https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/diffdiff_tomato_canny.jpeg")
|
||||||
@@ -322,7 +322,7 @@ DIFFDIFF_AUTO_BLOCKS.insert("controlnet_input",StableDiffusionXLControlNetAutoIn
|
|||||||
```py
|
```py
|
||||||
dd_auto_blocks = SequentialPipelineBlocks.from_blocks_dict(DIFFDIFF_AUTO_BLOCKS)
|
dd_auto_blocks = SequentialPipelineBlocks.from_blocks_dict(DIFFDIFF_AUTO_BLOCKS)
|
||||||
dd_pipeline = dd_auto_blocks.init_pipeline("YiYiXu/modular-demo-auto", collection="diffdiff")
|
dd_pipeline = dd_auto_blocks.init_pipeline("YiYiXu/modular-demo-auto", collection="diffdiff")
|
||||||
dd_pipeline.load_default_components(torch_dtype=torch.float16)
|
dd_pipeline.load_components(torch_dtype=torch.float16)
|
||||||
```
|
```
|
||||||
|
|
||||||
## 分享
|
## 分享
|
||||||
@@ -342,5 +342,5 @@ from diffusers.modular_pipelines import ModularPipeline, ComponentsManager
|
|||||||
components = ComponentsManager()
|
components = ComponentsManager()
|
||||||
|
|
||||||
diffdiff_pipeline = ModularPipeline.from_pretrained("YiYiXu/modular-diffdiff-0704", trust_remote_code=True, components_manager=components, collection="diffdiff")
|
diffdiff_pipeline = ModularPipeline.from_pretrained("YiYiXu/modular-diffdiff-0704", trust_remote_code=True, components_manager=components, collection="diffdiff")
|
||||||
diffdiff_pipeline.load_default_components(torch_dtype=torch.float16)
|
diffdiff_pipeline.load_components(torch_dtype=torch.float16)
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -1409,7 +1409,7 @@ class LoopSequentialPipelineBlocks(ModularPipelineBlocks):
|
|||||||
# YiYi TODO:
|
# YiYi TODO:
|
||||||
# 1. look into the serialization of modular_model_index.json, make sure the items are properly ordered like model_index.json (currently a mess)
|
# 1. look into the serialization of modular_model_index.json, make sure the items are properly ordered like model_index.json (currently a mess)
|
||||||
# 2. do we need ConfigSpec? the are basically just key/val kwargs
|
# 2. do we need ConfigSpec? the are basically just key/val kwargs
|
||||||
# 3. imnprove docstring and potentially add validator for methods where we accpet kwargs to be passed to from_pretrained/save_pretrained/load_default_components(), load_components()
|
# 3. imnprove docstring and potentially add validator for methods where we accpet kwargs to be passed to from_pretrained/save_pretrained/load_components()
|
||||||
class ModularPipeline(ConfigMixin, PushToHubMixin):
|
class ModularPipeline(ConfigMixin, PushToHubMixin):
|
||||||
"""
|
"""
|
||||||
Base class for all Modular pipelines.
|
Base class for all Modular pipelines.
|
||||||
@@ -1478,7 +1478,7 @@ class ModularPipeline(ConfigMixin, PushToHubMixin):
|
|||||||
- Components with default_creation_method="from_config" are created immediately, its specs are not included
|
- Components with default_creation_method="from_config" are created immediately, its specs are not included
|
||||||
in config dict and will not be saved in `modular_model_index.json`
|
in config dict and will not be saved in `modular_model_index.json`
|
||||||
- Components with default_creation_method="from_pretrained" are set to None and can be loaded later with
|
- Components with default_creation_method="from_pretrained" are set to None and can be loaded later with
|
||||||
`load_default_components()`/`load_components()`
|
`load_components()` (with or without specific component names)
|
||||||
- The pipeline's config dict is populated with component specs (only for from_pretrained components) and
|
- The pipeline's config dict is populated with component specs (only for from_pretrained components) and
|
||||||
config values, which will be saved as `modular_model_index.json` during `save_pretrained`
|
config values, which will be saved as `modular_model_index.json` during `save_pretrained`
|
||||||
- The pipeline's config dict is also used to store the pipeline blocks's class name, which will be saved as
|
- The pipeline's config dict is also used to store the pipeline blocks's class name, which will be saved as
|
||||||
@@ -1541,20 +1541,6 @@ class ModularPipeline(ConfigMixin, PushToHubMixin):
|
|||||||
params[input_param.name] = input_param.default
|
params[input_param.name] = input_param.default
|
||||||
return params
|
return params
|
||||||
|
|
||||||
def load_default_components(self, **kwargs):
|
|
||||||
"""
|
|
||||||
Load from_pretrained components using the loading specs in the config dict.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
**kwargs: Additional arguments passed to `from_pretrained` method, e.g. torch_dtype, cache_dir, etc.
|
|
||||||
"""
|
|
||||||
names = [
|
|
||||||
name
|
|
||||||
for name in self._component_specs.keys()
|
|
||||||
if self._component_specs[name].default_creation_method == "from_pretrained"
|
|
||||||
]
|
|
||||||
self.load_components(names=names, **kwargs)
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@validate_hf_hub_args
|
@validate_hf_hub_args
|
||||||
def from_pretrained(
|
def from_pretrained(
|
||||||
@@ -1682,8 +1668,8 @@ class ModularPipeline(ConfigMixin, PushToHubMixin):
|
|||||||
- non from_pretrained components are created during __init__ and registered as the object itself
|
- non from_pretrained components are created during __init__ and registered as the object itself
|
||||||
- Components are updated with the `update_components()` method: e.g. loader.update_components(unet=unet) or
|
- Components are updated with the `update_components()` method: e.g. loader.update_components(unet=unet) or
|
||||||
loader.update_components(guider=guider_spec)
|
loader.update_components(guider=guider_spec)
|
||||||
- (from_pretrained) Components are loaded with the `load_default_components()` method: e.g.
|
- (from_pretrained) Components are loaded with the `load_components()` method: e.g.
|
||||||
loader.load_default_components(names=["unet"])
|
loader.load_components(names=["unet"]) or loader.load_components() to load all default components
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
**kwargs: Keyword arguments where keys are component names and values are component objects.
|
**kwargs: Keyword arguments where keys are component names and values are component objects.
|
||||||
@@ -1995,13 +1981,14 @@ class ModularPipeline(ConfigMixin, PushToHubMixin):
|
|||||||
self.register_to_config(**config_to_register)
|
self.register_to_config(**config_to_register)
|
||||||
|
|
||||||
# YiYi TODO: support map for additional from_pretrained kwargs
|
# YiYi TODO: support map for additional from_pretrained kwargs
|
||||||
# YiYi/Dhruv TODO: consolidate load_components and load_default_components?
|
def load_components(self, names: Optional[Union[List[str], str]] = None, **kwargs):
|
||||||
def load_components(self, names: Union[List[str], str], **kwargs):
|
|
||||||
"""
|
"""
|
||||||
Load selected components from specs.
|
Load selected components from specs.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
names: List of component names to load; by default will not load any components
|
names: List of component names to load. If None, will load all components with
|
||||||
|
default_creation_method == "from_pretrained". If provided as a list or string, will load only the
|
||||||
|
specified components.
|
||||||
**kwargs: additional kwargs to be passed to `from_pretrained()`.Can be:
|
**kwargs: additional kwargs to be passed to `from_pretrained()`.Can be:
|
||||||
- a single value to be applied to all components to be loaded, e.g. torch_dtype=torch.bfloat16
|
- a single value to be applied to all components to be loaded, e.g. torch_dtype=torch.bfloat16
|
||||||
- a dict, e.g. torch_dtype={"unet": torch.bfloat16, "default": torch.float32}
|
- a dict, e.g. torch_dtype={"unet": torch.bfloat16, "default": torch.float32}
|
||||||
@@ -2009,7 +1996,13 @@ class ModularPipeline(ConfigMixin, PushToHubMixin):
|
|||||||
`variant`, `revision`, etc.
|
`variant`, `revision`, etc.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
if isinstance(names, str):
|
if names is None:
|
||||||
|
names = [
|
||||||
|
name
|
||||||
|
for name in self._component_specs.keys()
|
||||||
|
if self._component_specs[name].default_creation_method == "from_pretrained"
|
||||||
|
]
|
||||||
|
elif isinstance(names, str):
|
||||||
names = [names]
|
names = [names]
|
||||||
elif not isinstance(names, list):
|
elif not isinstance(names, list):
|
||||||
raise ValueError(f"Invalid type for names: {type(names)}")
|
raise ValueError(f"Invalid type for names: {type(names)}")
|
||||||
|
|||||||
@@ -67,7 +67,7 @@ class SDXLModularTests:
|
|||||||
|
|
||||||
def get_pipeline(self, components_manager=None, torch_dtype=torch.float32):
|
def get_pipeline(self, components_manager=None, torch_dtype=torch.float32):
|
||||||
pipeline = self.pipeline_blocks_class().init_pipeline(self.repo, components_manager=components_manager)
|
pipeline = self.pipeline_blocks_class().init_pipeline(self.repo, components_manager=components_manager)
|
||||||
pipeline.load_default_components(torch_dtype=torch_dtype)
|
pipeline.load_components(torch_dtype=torch_dtype)
|
||||||
return pipeline
|
return pipeline
|
||||||
|
|
||||||
def get_dummy_inputs(self, device, seed=0):
|
def get_dummy_inputs(self, device, seed=0):
|
||||||
@@ -158,7 +158,7 @@ class SDXLModularIPAdapterTests:
|
|||||||
blocks = self.pipeline_blocks_class()
|
blocks = self.pipeline_blocks_class()
|
||||||
_ = blocks.sub_blocks.pop("ip_adapter")
|
_ = blocks.sub_blocks.pop("ip_adapter")
|
||||||
pipe = blocks.init_pipeline(self.repo)
|
pipe = blocks.init_pipeline(self.repo)
|
||||||
pipe.load_default_components(torch_dtype=torch.float32)
|
pipe.load_components(torch_dtype=torch.float32)
|
||||||
pipe = pipe.to(torch_device)
|
pipe = pipe.to(torch_device)
|
||||||
pipe.set_progress_bar_config(disable=None)
|
pipe.set_progress_bar_config(disable=None)
|
||||||
cross_attention_dim = pipe.unet.config.get("cross_attention_dim")
|
cross_attention_dim = pipe.unet.config.get("cross_attention_dim")
|
||||||
|
|||||||
@@ -343,7 +343,7 @@ class ModularPipelineTesterMixin:
|
|||||||
with tempfile.TemporaryDirectory() as tmpdirname:
|
with tempfile.TemporaryDirectory() as tmpdirname:
|
||||||
base_pipe.save_pretrained(tmpdirname)
|
base_pipe.save_pretrained(tmpdirname)
|
||||||
pipe = ModularPipeline.from_pretrained(tmpdirname).to(torch_device)
|
pipe = ModularPipeline.from_pretrained(tmpdirname).to(torch_device)
|
||||||
pipe.load_default_components(torch_dtype=torch.float32)
|
pipe.load_components(torch_dtype=torch.float32)
|
||||||
pipe.to(torch_device)
|
pipe.to(torch_device)
|
||||||
|
|
||||||
pipes.append(pipe)
|
pipes.append(pipe)
|
||||||
|
|||||||
Reference in New Issue
Block a user