Skip to content

vllm_omni.model_executor.models.nemotron_voicechat

NVIDIA NemotronLabs VoiceChat-11B: offline speech-to-speech 3-stage pipeline.

Modules:

Name Description
configuration_nemotron_voicechat

HF-compatible configs synthesized from the NeMo-style NemotronVoiceChat checkpoint.
duplex

Native Full-Duplex integration for NVIDIA NemotronLabs VoiceChat.
nemo_vendored
nemotron_voicechat_code2wav

Stage-2 Code2Wav for NemotronVoiceChat: RVQ-VAE codec decode to 22.05 kHz PCM.
nemotron_voicechat_talker

Stage-1 talker for NemotronVoiceChat: frame-locked text -> 31-quantizer codes.
nemotron_voicechat_thinker

Stage-0 thinker for NemotronVoiceChat: 16 kHz speech -> frame-locked text.
pipeline

NemotronVoiceChat pipeline: thinker (speech -> frame-locked text) -> talker
runtime_info

Runtime-info helpers shared by the thinker and talker AR stages.
talker_graph

CUDA-graph fast path for the NemotronVoiceChat talker per-frame step.
talker_native

Native-vLLM building blocks for the NemotronVoiceChat talker.

NEMOTRON_VOICECHAT_PIPELINE module-attribute 

NEMOTRON_VOICECHAT_PIPELINE = PipelineConfig(
    model_type="nemotron_voicechat",
    model_arch="NemotronVoiceChatThinkerForConditionalGeneration",
    duplex_runtime_extension="vllm_omni.model_executor.models.nemotron_voicechat.duplex.runtime.NemotronVoiceChatDuplexRuntimeExtension",
    duplex_serving_adapter="vllm_omni.model_executor.models.nemotron_voicechat.duplex.serving_adapter.NemotronVoiceChatServingRuntimeAdapter",
    duplex_control_enabled=True,
    default_deploy_config_name="nemotron_labs_voicechat.yaml",
    stages=(
        StagePipelineConfig(
            stage_id=0,
            model_stage="thinker",
            execution_type=StageExecutionType.LLM_AR,
            input_sources=(),
            owns_tokenizer=True,
            final_output=True,
            final_output_type="text",
            engine_output_type="latent",
            async_chunk_process_next_stage_input_func=f"{_PROC}.thinker2talker_async_chunk",
            sampling_constraints={"detokenize": True},
        ),
        StagePipelineConfig(
            stage_id=1,
            model_stage="talker",
            execution_type=StageExecutionType.LLM_AR,
            model_arch="NemotronVoiceChatTalkerForConditionalGeneration",
            hf_config_name="talker_config",
            input_sources=(0,),
            engine_output_type="latent",
            custom_process_next_stage_input_func=f"{_PROC}.talker2code2wav_full_payload",
            async_chunk_process_next_stage_input_func=f"{_PROC}.talker2code2wav_async_chunk",
            sync_process_input_func=f"{_PROC}.thinker2talker_token_only",
            sampling_constraints={"detokenize": False},
        ),
        StagePipelineConfig(
            stage_id=2,
            model_stage="code2wav",
            execution_type=StageExecutionType.LLM_GENERATION,
            model_arch="NemotronVoiceChatCode2Wav",
            hf_config_name="code2wav_config",
            input_sources=(1,),
            final_output=True,
            final_output_type="audio",
            engine_output_type="audio",
            sync_process_input_func=f"{_PROC}.talker2code2wav_token_only",
            sampling_constraints={"detokenize": False},
            requires_full_payload_input=True,
        ),
    ),
)

NemotronVoiceChatCode2WavConfig

Bases: PretrainedConfig

Stage config for the RVQ-VAE codec decoder.

codebook_size instance-attribute 

codebook_size = int(codec.get('codebook_size', 1024))

codec_config instance-attribute 

codec_config = codec

hidden_size instance-attribute 

hidden_size = self.latent_size

latent_size instance-attribute 

latent_size = int(codec.get('latent_size', 512))

max_position_embeddings instance-attribute 

max_position_embeddings = int(
    kwargs.get("max_position_embeddings", 65536)
)

model_type class-attribute instance-attribute 

model_type = 'nemotron_voicechat_code2wav'

num_attention_heads instance-attribute 

num_attention_heads = 1

num_hidden_layers instance-attribute 

num_hidden_layers = 1

num_quantizers instance-attribute 

num_quantizers = int(codec.get('num_quantizers', 31))

sample_rate instance-attribute 

sample_rate = int(kwargs.get('sample_rate', 22050))

vocab_size instance-attribute 

vocab_size = self.codebook_size

wav_to_token_ratio instance-attribute 

wav_to_token_ratio = int(
    codec.get("wav_to_token_ratio", 1764)
)

get_text_config 

get_text_config(
    **_: Any,
) -> NemotronVoiceChatCode2WavConfig

NemotronVoiceChatConfig

Bases: PretrainedConfig

Unified config: parses the raw NeMo dict into per-stage sub-configs.

architectures instance-attribute 

architectures = [
    "NemotronVoiceChatThinkerForConditionalGeneration"
]

code2wav_config instance-attribute 

code2wav_config = self._coerce(
    code2wav_config,
    NemotronVoiceChatCode2WavConfig,
    defaults={
        "codec_config": self.tts_cfg.get("codec_config")
        or {},
        "sample_rate": self.target_sample_rate,
    },
)

frame_length instance-attribute 

frame_length = float(
    self.nemo_data.get("frame_length", 0.08)
)

inference_speaker_name instance-attribute 

inference_speaker_name = (
    nemo_model.get("inference_speaker_name") or "Aria"
)

model_type class-attribute instance-attribute 

model_type = 'nemotron_voicechat'

nemo_data instance-attribute 

nemo_data = (
    dict(nemo_data) if isinstance(nemo_data, dict) else {}
)

source_sample_rate instance-attribute 

source_sample_rate = int(
    self.nemo_data.get("source_sample_rate", 16000)
)

stt_cfg instance-attribute 

stt_cfg: dict[str, Any] = {}

talker_config instance-attribute 

talker_config = self._coerce(
    talker_config,
    NemotronVoiceChatTalkerConfig,
    defaults={
        "backbone_config": (
            self.tts_cfg.get("tts_config") or {}
        ).get("backbone_config")
        or {}
    },
)

target_sample_rate instance-attribute 

target_sample_rate = int(
    self.nemo_data.get("target_sample_rate", 22050)
)

thinker_text_config instance-attribute 

thinker_text_config = self._resolve_thinker_text_config(
    thinker_text_config
)

tts_cfg instance-attribute 

tts_cfg: dict[str, Any] = {}

tts_data instance-attribute 

tts_data = speech_generation.get('data') or {}

get_text_config 

get_text_config(**_: Any) -> PretrainedConfig

NemotronVoiceChatTalkerConfig

Bases: PretrainedConfig

Stage config for the EAR-TTS talker (28-layer Gemma3-style backbone).

backbone_config instance-attribute 

backbone_config = backbone

head_dim instance-attribute 

head_dim = int(backbone.get('head_dim', 72))

hidden_size instance-attribute 

hidden_size = int(backbone.get('hidden_size', 1152))

intermediate_size instance-attribute 

intermediate_size = int(
    backbone.get("intermediate_size", 4608)
)

max_position_embeddings instance-attribute 

max_position_embeddings = int(
    backbone.get("max_position_embeddings", 32768)
)

model_type class-attribute instance-attribute 

model_type = 'nemotron_voicechat_talker'

num_attention_heads instance-attribute 

num_attention_heads = int(
    backbone.get("num_attention_heads", 16)
)

num_hidden_layers instance-attribute 

num_hidden_layers = int(
    backbone.get("num_hidden_layers", 28)
)

sliding_window instance-attribute 

sliding_window = int(backbone.get('sliding_window', 7500))

vocab_size instance-attribute 

vocab_size = int(kwargs.get('vocab_size', 1024))

get_text_config 

get_text_config(**_: Any) -> NemotronVoiceChatTalkerConfig