阿里千问发布 Qwen3.8-Omni-Flash 及实时版

api_key = os . environ . get ( "DASHSCOPE_API_KEY" ) if not api_key : raise ValueError ( "DASHSCOPE_API_KEY is required. " "Set it via: export DASHSCOPE_API_KEY='your-api-key'" )

client = OpenAI ( api_key = api_key , base_url = os . environ . get ( "DASHSCOPE_BASE_URL" , "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" , ), )

messages = [ { "role" : "user" , "content" : [ { "type" : "image_url" , "image_url" : { "url" : "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241022/emyrja/dog_and_girl.jpeg" }, }, { "type" : "input_audio" , "input_audio" : { "data" : "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20250211/tixcef/cherry.wav" , "format" : "wav" }, }, { "type" : "text" , "text" : "Please describe the image and tell me what is being said in the audio." }, ], }, ]

completion = client . chat . completions . create ( model = "qwen3.8-omni-flash" , messages = messages , extra_body = { "enable_thinking" : True , # "preserve_thinking": True, }, reasoning_effort = "xhigh" , # supported levels are xhigh, medium, and low stream = True , )

reasoning_content = "" answer_content = "" is_answering = False print ( " \n " + "=" * 20 + "Reasoning" + "=" * 20 + " \n " )

for chunk in completion : if not chunk . choices : print ( " \n Usage:" ) print ( chunk . usage ) continue

delta = chunk . choices [ 0 ] . delta

if hasattr ( delta , "reasoning_content" ) and delta . reasoning_content is not None : if not is_answering : print ( delta . reasoning_content , end = "" , flush = True ) reasoning_content += delta . reasoning_content

if hasattr ( delta , "content" ) and delta . content : if not is_answering : print ( " \n " + "=" * 20 + "Answer" + "=" * 20 + " \n " ) is_answering = True print ( delta . content , end = "" , flush = True ) answer_content += delta . content Best Practices for Controllable Audio-Visual Captioning # Content organization and level of detail: Organize video descriptions, OCR extraction, audio descriptions, and speech transcripts into sections covering visuals, speech, music, sound effects, and ambient sounds. Preserve the original text, speaker identities, and corresponding time ranges to make retrieval and verification easier. Specify a target length in the prompt to control detail, for example, Describe the video in approximately 2000–3000 words. , and adjust it to the video’s duration and information density. Prompt Example Provide a detailed description of the video.

Make sure your description covers every one of the following dimensions:

Visual - Subjects and characters: appearance, clothing, gender/age cues, identity, distinctive features - Actions and events in chronological order, and how the scene evolves over time - Setting and background: location, environment, time of day - Spatial layout and relations between subjects/objects; counts and quantities - On-screen text: captions, titles, subtitles, logos, UI — exact content and appearance - Visual style: colors, lighting, camera shots, angles, and camera movement

Audio - Speech: the exact spoken content, transcribed verbatim - Speakers: who is speaking (mapped to the on-screen person or voice-over), with accent, tone, gender/age cues - Speaking state: prosody, emotion, volume, and speaking style - Music: presence, genre/mood, and lyrics if any - Sound effects and ambient/background sounds - Non-speech vocalizations: laughter, crying, applause, etc.

Audio-visual correspondence - Which speech or sound aligns with which on-screen person or visual event - The timing of each event, expressed with timestamps

It should explicitly include three sections:

1. A structured chronological storyline of **every noticeable audio and visual details** 2. A structured list of all visible text. For each text element, include start timestamp, end timestamp, the exact text content, the appearance characteristics. If no text appears, explicitly state so. 3. A structured speech-to-text transcription, include speaker (corresponding to the character or voice-over in Section 1, including their accent and tone), exact spoken content, start timestamp, end timestamp, and speaking state (prosody, emotion, and style). If no speech appears, explicitly state so.

Aside from these three required sections, you are free to organize any additional content in any way you find helpful. This additional content can include global information about the entire video or localized information about specific moments. You may choose the topic of this extra content freely.

Rules:

- Add as much descriptive detail as possible. - Do not use Markdown bold formatting. - Carefully look at frames and listen to the audio, making sure no detail is overlooked.

Output Format: ``` ## Storyline

- Speaker: State: Content: “ ”

... ``` Structured output and schema constraints: Include task instructions and a complete JSON Schema in the prompt, specifying field meanings, types, required fields, and constraints to support programmatic parsing and downstream use. Prompt Example Describe the audio and visual content in detail in English, organized into scenes and events, following the JSON Schema below. All timestamps must be relative to the beginning of the video. End times must not precede start times or exceed the video duration. Each event must fall within the time range of its parent scene. Include only information directly supported by the audio or video. Do not guess or invent details. Do not infer causality merely because a sound and an action occur at the same time. Return only valid JSON, without Markdown fences or commentary.

JSON Schema: { "$defs": { "Event": { "additionalProperties": false, "properties": { "time_range": { "$ref": "#/$defs/TimeRange", "description": "Time range of the event" }, "participants": { "description": "People, animals, or objects involved, named by observable features; use consistent names for the same participant", "items": { "type": "string" }, "title": "Participants", "type": "array" }, "action": { "description": "Specific actions, interactions, and observable outcomes", "title": "Action", "type": "string" }, "sounds": { "description": "Sounds heard during the event; use an empty list if none are discernible", "items": { "type": "string" }, "title": "Sounds", "type": "array" } }, "required": [ "time_range", "participants", "action", "sounds" ], "title": "Event", "type": "object" }, "Scene": { "additionalProperties": false, "properties": { "time_range": { "$ref": "#/$defs/TimeRange", "description": "Time range of the scene" }, "setting": { "description": "Environment, spatial layout, and main visual features", "title": "Setting", "type": "string" }, "events": { "description": "Events in chronological order; use an empty list if there are none", "items": { "$ref": "#/$defs/Event" }, "title": "Events", "type": "array" } }, "required": [ "time_range", "setting", "events" ], "title": "Scene", "type": "object" }, "TimeRange": { "additionalProperties": false, "properties": { "start_seconds": { "description": "Start time in seconds relative to the beginning of the video", "minimum": 0, "title": "Start Seconds", "type": "number" }, "end_seconds": { "description": "End time in seconds; must not precede the start time", "minimum": 0, "title": "End Seconds", "type": "number" } }, "required": [ "start_seconds", "end_seconds" ], "title": "TimeRange", "type": "object" } }, "additionalProperties": false, "properties": { "summary": { "description": "An overview of the main content of the video", "title": "Summary", "type": "string" }, "scenes": { "description": "Scenes in chronological order; group continuous footage with a consistent setting into one scene", "items": { "$ref": "#/$defs/Scene" }, "title": "Scenes", "type": "array" } }, "required": [ "summary", "scenes" ], "title": "CaptionResult", "type": "object" } Installing Qwen-MM-Plugins in an Agent Harness # Qwen-MM-Plugins is a multimodal plugin suite for agent harnesses. It gives agents the ability to understand images, audio, video, and documents, as well as maintain memory for long-form video and create content. It supports agent harnesses including Codex, Claude Code, Qwen Code, Gemini CLI, Qoder, CodeBuddy, and OpenClaw. We also welcome contributions from community developers. You can enter the following request directly in your usual office agent: Help me install the core, api, and omni-related plugins from https://github.com/QwenLM/Qwen-MM-Plugins. Alternatively, install from the command line: # Run the official guided installer: curl -fsSL https://raw.githubusercontent.com/QwenLM/Qwen-MM-Plugins/main/install.sh | bash In the menu, select: Install The agent harness you use The Omni plugins you need Plugin Capability core Read images, video frames, PDFs, Office documents, code, data, and 3D files api Image understanding, OCR, object localization, audio-visual transcription, speaker diarization, event analysis, and image segmentation omni-chatcut Create MVs, long-form film commentary, and video speech translations omni-video2note Turn video tutorials into PDF notes with key screenshots omni-skill-creator Turn instructional videos, screen recordings, or operation demonstrations into reusable Agent Skill.md files omni-memory Build memory for people, dialogue, sounds, and events in long-form videos After installation, restart the agent harness or create a new task. @song.mp3 Generate a complete MV based on the song's rhythm and content. @short_drama.mp4 Translate the video into English while preserving the original speakers' voice characteristics where possible. @movie.mp4 Create a film commentary video with Chinese narration and subtitles.

@weekly_report_sop.mp4 Turn this screen recording of writing a weekly report into a Skill.md file. @tutorial.mp4 Turn the tutorial into PDF notes with key screenshots, timestamps, and step-by-step instructions. @documentary.mp4 Build audio-visual memory that records people, dialogue, sounds, and important events. Getting Started with Qwen3.8-Omni-Flash-Realtime # API Usage # Qwen3.8-Omni-Flash-Realtime supports connections over WebSocket and WebRTC. Running the basic examples below opens the camera and microphone for a real-time audio-visual conversation. We recommend using headphones. # Run pip install websocket-client pyaudio dashscope opencv-python -U to install dependencies """ Environment variables: DASHSCOPE_API_KEY: Your API Key from https://platform.qianwenai.com/home DASHSCOPE_BASE_URL: (optional) Base URL for realtime API. - Beijing: wss://dashscope.aliyuncs.com/api-ws/v1/realtime - Singapore: wss://dashscope-intl.aliyuncs.com/api-ws/v1/realtime """ import os import base64 import time import pyaudio import cv2 from dashscope.audio.qwen_omni import MultiModality , AudioFormat , OmniRealtimeCallback , OmniRealtimeConversation import dashscope

url = f 'wss://dashscope-intl.aliyuncs.com/api-ws/v1/realtime' dashscope . api_key = os . getenv ( 'DASHSCOPE_API_KEY' ) # Determine the voice voice = 'Tina' # Determine the model model = 'qwen3.8-omni-flash-realtime' # Determine the model role instructions = "You are Qwen-Omni, a helpful assistant." video_fps = 1 video_size = ( 1280 , 720 ) class SimpleCallback ( OmniRealtimeCallback ): def __init__ ( self , pya ): self . pya = pya self . out = None def on_open ( self ): # Initialize audio output stream self . out = self . pya . open ( format = pyaudio . paInt16 , channels = 1 , rate = 24000 , output = True ) def on_event ( self , response ): if response [ 'type' ] == 'response.audio.delta' : # Play audio self . out . write ( base64 . b64decode ( response [ 'delta' ])) elif response [ 'type' ] == 'conversation.item.input_audio_transcription.delta' : # Streaming preview: text is the confirmed prefix, stash is the confirmed suffix preview = response . get ( 'text' , '' ) + response . get ( 'stash' , '' ) print ( f " \r [User] { preview } " , end = '' , flush = True ) elif response [ 'type' ] == 'conversation.item.input_audio_transcription.completed' : # Transcription completed, print the final text and a new line print ( f " \r [User] { response [ 'transcript' ] } " ) elif response [ 'type' ] == 'response.audio_transcript.done' : # Print the assistant's response text print ( f "[LLM] { response [ 'transcript' ] } " )