Compare commits

...

5 Commits

Author SHA1 Message Date
sayakpaul
044a7f60d3 style. 2026-02-03 16:56:20 +05:30
sayakpaul
7b0f324371 refine. 2026-02-03 15:18:41 +05:30
sayakpaul
efd3baaa97 simplify structure. 2026-02-03 14:49:14 +05:30
sayakpaul
32690e200c start better template for modular pipeline card. 2026-02-03 13:37:25 +05:30
songkey
b712042da1 [Flux2] Fix LoRA loading for Flux2 Klein by adaptively enumerating transformer blocks (#13030)
* Resolve Flux2 Klein 4B/9B LoRA loading errors

* Apply style fixes

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Sayak Paul <spsayakpaul@gmail.com>
2026-02-02 20:36:19 +05:30
4 changed files with 209 additions and 8 deletions

View File

@@ -2321,8 +2321,14 @@ def _convert_non_diffusers_flux2_lora_to_diffusers(state_dict):
prefix = "diffusion_model."
original_state_dict = {k[len(prefix) :]: v for k, v in state_dict.items()}
num_double_layers = 8
num_single_layers = 48
num_double_layers = 0
num_single_layers = 0
for key in original_state_dict.keys():
if key.startswith("single_blocks."):
num_single_layers = max(num_single_layers, int(key.split(".")[1]) + 1)
elif key.startswith("double_blocks."):
num_double_layers = max(num_double_layers, int(key.split(".")[1]) + 1)
lora_keys = ("lora_A", "lora_B")
attn_types = ("img_attn", "txt_attn")

View File

@@ -34,6 +34,7 @@ from ..utils.dynamic_modules_utils import get_class_from_dynamic_module, resolve
from ..utils.hub_utils import load_or_create_model_card, populate_model_card
from .components_manager import ComponentsManager
from .modular_pipeline_utils import (
MODULAR_MODEL_CARD_TEMPLATE,
ComponentSpec,
ConfigSpec,
InputParam,
@@ -1734,6 +1735,159 @@ class ModularPipeline(ConfigMixin, PushToHubMixin):
)
return pipeline
def _generate_modular_model_card_content(self) -> Dict[str, Any]:
from .modular_pipeline_utils import format_components, format_configs
blocks_class_name = self.blocks.__class__.__name__
pipeline_name = blocks_class_name.replace("Blocks", " Pipeline")
description = self.blocks.description or "A modular diffusion pipeline."
# generate blocks architecture description
blocks_desc_parts = []
for i, (name, block) in enumerate(self.blocks.sub_blocks.items()):
block_class = block.__class__.__name__
block_desc = block.description.split("\n")[0] if block.description else ""
blocks_desc_parts.append(f"{i + 1}. **{name}** (`{block_class}`)")
if block_desc:
blocks_desc_parts.append(f" - {block_desc}")
# add sub-blocks if any
if hasattr(block, "sub_blocks") and block.sub_blocks:
for sub_name, sub_block in block.sub_blocks.items():
sub_class = sub_block.__class__.__name__
sub_desc = sub_block.description.split("\n")[0] if sub_block.description else ""
blocks_desc_parts.append(f" - *{sub_name}*: `{sub_class}`")
if sub_desc:
blocks_desc_parts.append(f" - {sub_desc}")
blocks_description = "\n".join(blocks_desc_parts) if blocks_desc_parts else "No blocks defined."
components = self.blocks.expected_components
if components:
components_str = format_components(components, indent_level=0, add_empty_lines=False)
# remove the "Components:" header since template has its own
components_description = components_str.replace("Components:\n", "").strip()
if components_description:
# Convert to enumerated list
lines = [line.strip() for line in components_description.split("\n") if line.strip()]
enumerated_lines = [f"{i + 1}. {line}" for i, line in enumerate(lines)]
components_description = "\n".join(enumerated_lines)
else:
components_description = "No specific components required."
else:
components_description = "No specific components required. Components can be loaded dynamically."
configs = self.blocks.expected_configs
configs_section = ""
if configs:
configs_str = format_configs(configs, indent_level=0, add_empty_lines=False)
configs_description = configs_str.replace("Configs:\n", "").strip()
if configs_description:
configs_section = f"\n\n## Configuration Parameters\n\n{configs_description}"
inputs = self.blocks.inputs
outputs = self.blocks.outputs
# format inputs as markdown list
inputs_parts = []
required_inputs = [inp for inp in inputs if inp.required]
optional_inputs = [inp for inp in inputs if not inp.required]
if required_inputs:
inputs_parts.append("**Required:**\n")
for inp in required_inputs:
if hasattr(inp.type_hint, "__name__"):
type_str = inp.type_hint.__name__
elif inp.type_hint is not None:
type_str = str(inp.type_hint).replace("typing.", "")
else:
type_str = "Any"
desc = inp.description or "No description provided"
inputs_parts.append(f"- `{inp.name}` (`{type_str}`): {desc}")
if optional_inputs:
if required_inputs:
inputs_parts.append("")
inputs_parts.append("**Optional:**\n")
for inp in optional_inputs:
if hasattr(inp.type_hint, "__name__"):
type_str = inp.type_hint.__name__
elif inp.type_hint is not None:
type_str = str(inp.type_hint).replace("typing.", "")
else:
type_str = "Any"
desc = inp.description or "No description provided"
default_str = f", default: `{inp.default}`" if inp.default is not None else ""
inputs_parts.append(f"- `{inp.name}` (`{type_str}`){default_str}: {desc}")
inputs_description = "\n".join(inputs_parts) if inputs_parts else "No specific inputs defined."
# format outputs as markdown list
outputs_parts = []
for out in outputs:
if hasattr(out.type_hint, "__name__"):
type_str = out.type_hint.__name__
elif out.type_hint is not None:
type_str = str(out.type_hint).replace("typing.", "")
else:
type_str = "Any"
desc = out.description or "No description provided"
outputs_parts.append(f"- `{out.name}` (`{type_str}`): {desc}")
outputs_description = "\n".join(outputs_parts) if outputs_parts else "Standard pipeline outputs."
trigger_inputs_section = ""
if hasattr(self.blocks, "trigger_inputs") and self.blocks.trigger_inputs:
trigger_inputs_list = sorted([t for t in self.blocks.trigger_inputs if t is not None])
if trigger_inputs_list:
trigger_inputs_str = ", ".join(f"`{t}`" for t in trigger_inputs_list)
trigger_inputs_section = f"""
### Conditional Execution
This pipeline contains blocks that are selected at runtime based on inputs:
- **Trigger Inputs**: {trigger_inputs_str}
"""
# generate tags based on pipeline characteristics
tags = ["modular-diffusers", "diffusers"]
if hasattr(self.blocks, "model_name") and self.blocks.model_name:
tags.append(self.blocks.model_name)
if hasattr(self.blocks, "trigger_inputs") and self.blocks.trigger_inputs:
triggers = self.blocks.trigger_inputs
if any(t in triggers for t in ["mask", "mask_image"]):
tags.append("inpainting")
if any(t in triggers for t in ["image", "image_latents"]):
tags.append("image-to-image")
if any(t in triggers for t in ["control_image", "controlnet_cond"]):
tags.append("controlnet")
if not any(t in triggers for t in ["image", "mask", "image_latents", "mask_image"]):
tags.append("text-to-image")
else:
tags.append("text-to-image")
block_count = len(self.blocks.sub_blocks)
model_description = f"""This is a modular diffusion pipeline built with 🧨 Diffusers' modular pipeline framework.
**Pipeline Type**: {blocks_class_name}
**Description**: {description}
This pipeline uses a {block_count}-block architecture that can be customized and extended."""
return {
"pipeline_name": pipeline_name,
"model_description": model_description,
"blocks_description": blocks_description,
"components_description": components_description,
"configs_section": configs_section,
"inputs_description": inputs_description,
"outputs_description": outputs_description,
"trigger_inputs_section": trigger_inputs_section,
"tags": tags,
}
def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs):
"""
Save the pipeline to a directory. It does not save components, you need to save them separately.
@@ -1753,9 +1907,19 @@ class ModularPipeline(ConfigMixin, PushToHubMixin):
repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1])
repo_id = create_repo(repo_id, exist_ok=True, private=private, token=token).repo_id
# Generate modular pipeline card content
card_content = self._generate_modular_model_card_content()
# Create a new empty model card and eventually tag it
model_card = load_or_create_model_card(repo_id, token=token, is_pipeline=True)
model_card = populate_model_card(model_card)
model_card = load_or_create_model_card(
repo_id,
token=token,
is_pipeline=True,
model_description=MODULAR_MODEL_CARD_TEMPLATE.format(**card_content),
is_modular=True,
)
model_card = populate_model_card(model_card, tags=card_content["tags"])
model_card.save(os.path.join(save_directory, "README.md"))
# YiYi TODO: maybe order the json file to make it more readable: configs first, then components

View File

@@ -31,6 +31,30 @@ if is_torch_available():
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
# Template for modular pipeline model card description with placeholders
MODULAR_MODEL_CARD_TEMPLATE = """{model_description}
## Example Usage
[TODO]
## Pipeline Architecture
This modular pipeline is composed of the following blocks:
{blocks_description} {trigger_inputs_section}
## Model Components
{components_description} {configs_section}
## Input/Output Specification
### Inputs {inputs_description}
### Outputs {outputs_description}
"""
class InsertableDict(OrderedDict):
def insert(self, key, value, index):

View File

@@ -107,6 +107,7 @@ def load_or_create_model_card(
license: Optional[str] = None,
widget: Optional[List[dict]] = None,
inference: Optional[bool] = None,
is_modular: bool = False,
) -> ModelCard:
"""
Loads or creates a model card.
@@ -131,6 +132,8 @@ def load_or_create_model_card(
widget (`List[dict]`, *optional*): Widget to accompany a gallery template.
inference: (`bool`, optional): Whether to turn on inference widget. Helpful when using
`load_or_create_model_card` from a training script.
is_modular: (`bool`, optional): Boolean flag to denote if the model card is for a modular pipeline.
When True, uses model_description as-is without additional template formatting.
"""
if not is_jinja_available():
raise ValueError(
@@ -159,10 +162,14 @@ def load_or_create_model_card(
)
else:
card_data = ModelCardData()
component = "pipeline" if is_pipeline else "model"
if model_description is None:
model_description = f"This is the model card of a 🧨 diffusers {component} that has been pushed on the Hub. This model card has been automatically generated."
model_card = ModelCard.from_template(card_data, model_description=model_description)
if is_modular and model_description is not None:
model_card = ModelCard(model_description)
model_card.data = card_data
else:
component = "pipeline" if is_pipeline else "model"
if model_description is None:
model_description = f"This is the model card of a 🧨 diffusers {component} that has been pushed on the Hub. This model card has been automatically generated."
model_card = ModelCard.from_template(card_data, model_description=model_description)
return model_card