vllm_omni.diffusion.models.longcat_video ¶

audio_stride = 1

build_components_on_gpu = _as_bool(
    additional_config.get("build_components_on_gpu"), False
)

default_num_frames = 93

device = get_local_device()

do_classifier_free_guidance

dummy_run_num_frames: int = 1

model_dir = _ensure_local_dir(
    od_config.model,
    allow_patterns=_avatar_model_allow_patterns(
        self.use_int8
    ),
)

model_type = str(
    additional_config.get("model_type") or "avatar-v1.5"
)

num_distill_sample_steps

num_timesteps

od_config = od_config

resolution = str(
    additional_config.get("resolution") or "480p"
)

save_fps = 25

support_audio_input: bool = True

support_image_input: bool = True

use_distill = _as_bool(
    additional_config.get("use_distill"), True
)

use_int8 = _as_bool(additional_config.get("use_int8"), True)

video_processor = VideoProcessor(vae_scale_factor=8)

weights_sources = [
    DiffusersPipelineLoader.ComponentSource(
        model_or_path=str(self.model_dir),
        subfolder=dit_subfolder,
        revision=None,
        prefix="transformer.",
        fall_back_to_pt=False,
    )
]

active_loras = []

audio_proj = AudioProjModel(
    seq_len=audio_window,
    seq_len_vf=audio_window + vae_scale - 1,
    blocks=audio_block,
    channels=audio_channel,
    intermediate_dim=intermediate_dim,
    output_dim=output_dim,
    context_tokens=context_tokens,
)

audio_window = audio_window

blocks = nn.ModuleList(
    [
        LongCatAvatarSingleStreamBlock(
            hidden_size=hidden_size,
            num_heads=num_heads,
            mlp_ratio=mlp_ratio,
            adaln_tembed_dim=adaln_tembed_dim,
            enable_flashattn3=enable_flashattn3,
            enable_flashattn2=enable_flashattn2,
            enable_xformers=enable_xformers,
            enable_bsa=enable_bsa,
            bsa_params=bsa_params,
            cp_split_hw=self.cp_split_hw,
            output_dim=output_dim,
            audio_prenorm=audio_prenorm,
            class_range=class_range,
            class_interval=class_interval,
        )
        for _ in range(depth)
    ]
)

config = SimpleNamespace(
    in_channels=in_channels,
    out_channels=out_channels,
    hidden_size=hidden_size,
    depth=depth,
    num_heads=num_heads,
    caption_channels=caption_channels,
    mlp_ratio=mlp_ratio,
    adaln_tembed_dim=adaln_tembed_dim,
    frequency_embedding_size=frequency_embedding_size,
    patch_size=self.patch_size,
    enable_flashattn3=enable_flashattn3,
    enable_flashattn2=enable_flashattn2,
    enable_xformers=enable_xformers,
    enable_bsa=enable_bsa,
    bsa_params=bsa_params,
    cp_split_hw=None,
    text_tokens_zero_pad=text_tokens_zero_pad,
    audio_window=audio_window,
    audio_block=audio_block,
    audio_channel=audio_channel,
    intermediate_dim=intermediate_dim,
    output_dim=output_dim,
    context_tokens=context_tokens,
    vae_scale=vae_scale,
    audio_prenorm=audio_prenorm,
    class_range=class_range,
    class_interval=class_interval,
)

cp_split_hw = (1, 1)

dtype: dtype

final_layer = FinalLayer_FP32(
    hidden_size,
    np.prod(self.patch_size),
    out_channels,
    adaln_tembed_dim,
)

gradient_checkpointing = False

in_channels = in_channels

lora_dict = {}

out_channels = out_channels

patch_size = tuple(patch_size)

t_embedder = TimestepEmbedder(
    t_embed_dim=adaln_tembed_dim,
    frequency_embedding_size=frequency_embedding_size,
)

text_tokens_zero_pad = text_tokens_zero_pad

vae_scale = vae_scale

x_embedder = PatchEmbed3D(
    self.patch_size, in_channels, hidden_size
)

y_embedder = CaptionEmbedder(
    in_channels=caption_channels, hidden_size=hidden_size
)