vllm_omni.diffusion.models.lingbot_video ¶

T2I = 't2i'

T2V = 't2v'

TI2V = 'ti2v'

clean_latent: Tensor

vlm_image: Image

fps: int

guidance_scale: float

height: int

input_image: Image | None

mode: LingBotGenerationMode

negative_prompt: str

num_frames: int

num_inference_steps: int

output_type: str

prompt: str

shift: float

width: int

default_image_negative_prompt = (
    DEFAULT_NEGATIVE_PROMPT_IMAGE
)

default_negative_prompt = DEFAULT_NEGATIVE_PROMPT

device = get_local_device()

hidden_state_skip_layer = HIDDEN_STATE_SKIP_LAYER

img_prompt_template = IMG_PROMPT_TEMPLATE

max_outputs_per_prompt: int = 1

od_config = od_config

processor = Qwen3VLProcessor.from_pretrained(
    model,
    subfolder=processor_subfolder,
    local_files_only=local_files_only,
)

prompt_template = PROMPT_TEMPLATE

scheduler = FlowUniPCMultistepScheduler.from_pretrained(
    model,
    subfolder=scheduler_subfolder,
    local_files_only=local_files_only,
)

supports_step_execution: bool = False

text_encoder = (
    Qwen3VLForConditionalGeneration.from_pretrained(
        model,
        subfolder=text_encoder_subfolder,
        **text_encoder_kwargs,
    ).to(self.device)
)

token_length = TOKEN_LENGTH

transformer = (
    LingBotVideoTransformer3DModel.from_pretrained(
        model,
        subfolder=transformer_subfolder,
        torch_dtype=transformer_dtype,
        local_files_only=local_files_only,
    ).to(self.device)
)

vae = AutoencoderKLWan.from_pretrained(
    model,
    subfolder=vae_subfolder,
    torch_dtype=vae_dtype,
    local_files_only=local_files_only,
).to(self.device)

vae_scale_factor_spatial = 8

vae_scale_factor_temporal = 4

blocks = nn.ModuleList(
    [
        LingBotVideoBlock(
            hidden_size=hidden_size,
            num_attention_heads=num_attention_heads,
            intermediate_size=intermediate_size,
            norm_eps=norm_eps,
            qkv_bias=qkv_bias,
            out_bias=out_bias,
            num_experts=num_experts,
            num_experts_per_tok=num_experts_per_tok,
            moe_intermediate_size=moe_intermediate_size,
            decoder_sparse_step=decoder_sparse_step,
            mlp_only_layers=mlp_only_layers,
            n_shared_experts=n_shared_experts,
            score_func=score_func,
            norm_topk_prob=norm_topk_prob,
            n_group=n_group,
            topk_group=topk_group,
            routed_scaling_factor=routed_scaling_factor,
            layer_idx=i,
        )
        for i in range(depth)
    ]
)

norm_out = nn.LayerNorm(
    hidden_size, elementwise_affine=False, eps=norm_eps
)

norm_out_modulation = nn.Sequential(
    nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size)
)

patch_embedder = nn.Linear(
    in_channels * math.prod(patch_size),
    hidden_size,
    bias=patch_embed_bias,
)

proj_out = nn.Linear(
    hidden_size, math.prod(patch_size) * out_channels
)

rope = LingBotVideoRotaryEmbedding(
    axes_dims, axes_lens, rope_theta
)

text_embedder = LingBotVideoTextEmbedder(
    text_dim, hidden_size
)

time_embedder = TimestepEmbedding(
    freq_dim,
    hidden_size,
    act_fn="silu",
    sample_proj_bias=timestep_mlp_bias,
)

time_modulation = nn.Sequential(
    nn.SiLU(), nn.Linear(hidden_size, 6 * hidden_size)
)

time_proj = Timesteps(
    freq_dim, flip_sin_to_cos=True, downscale_freq_shift=0
)