From a2a07ade7bbfdadca19ab189ce94fbb5036ab108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=81=A5=E4=BB=99?= Date: Wed, 9 Sep 2026 11:16:38 +0800 Subject: [PATCH 1/2] feat(model/asr): support update input.context by continue-task --- .../audio/asr/recognition/Recognition.java | 53 +++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/alibaba/dashscope/audio/asr/recognition/Recognition.java b/src/main/java/com/alibaba/dashscope/audio/asr/recognition/Recognition.java index 2c17ca1..cce0811 100644 --- a/src/main/java/com/alibaba/dashscope/audio/asr/recognition/Recognition.java +++ b/src/main/java/com/alibaba/dashscope/audio/asr/recognition/Recognition.java @@ -37,12 +37,12 @@ public final class Recognition { private ApiServiceOption serviceOption; - private Emitter audioEmitter; + private Emitter streamDataEmitter; @SuperBuilder private static class AsyncCmdBuffer { @Builder.Default private boolean isStop = false; - private ByteBuffer audioFrame; + private Object streamData; } private final Queue cmdBuffer = new LinkedList<>(); @@ -62,22 +62,22 @@ private static class AsyncCmdBuffer { @SuperBuilder private static class RecognitionParamWithStream extends RecognitionParam { - @NonNull private Flowable audioStream; + @NonNull private Flowable streamData; @Override public Flowable getStreamingData() { - return audioStream.cast(Object.class); + return streamData; } public static RecognitionParamWithStream FromRecognitionParam( - RecognitionParam param, Flowable audioStream, String preRequestId) { + RecognitionParam param, Flowable streamData, String preRequestId) { RecognitionParamWithStream recognitionParamWithStream = RecognitionParamWithStream.builder() .parameters((param.getParameters())) .parameter("pre_task_id", preRequestId) .headers(param.getHeaders()) .format(param.getFormat()) - .audioStream(audioStream) + .streamData(streamData.cast(Object.class)) .disfluencyRemovalEnabled(param.isDisfluencyRemovalEnabled()) .model(param.getModel()) .sampleRate(param.getSampleRate()) @@ -217,7 +217,7 @@ public void call(RecognitionParam param, ResultCallback callb new InputRequiredException("Parameter invalid: ResultCallback is null")); } - Flowable audioFrames = + Flowable audioFrames = Flowable.create( emitter -> { synchronized (Recognition.this) { @@ -227,12 +227,12 @@ public void call(RecognitionParam param, ResultCallback callb emitter.onComplete(); return; } else { - emitter.onNext(buffer.audioFrame); + emitter.onNext(buffer.streamData); } } cmdBuffer.clear(); } - audioEmitter = emitter; + streamDataEmitter = emitter; } }, BackpressureStrategy.BUFFER); @@ -423,10 +423,33 @@ public void sendAudioFrame(ByteBuffer audioFrame) { new InputRequiredException( "State invalid: expect recognition state is started but " + state.getValue())); } - if (audioEmitter == null) { - cmdBuffer.add(AsyncCmdBuffer.builder().audioFrame(audioFrame).build()); + if (streamDataEmitter == null) { + cmdBuffer.add(AsyncCmdBuffer.builder().streamData(audioFrame).build()); + } else { + streamDataEmitter.onNext(audioFrame); + } + } + } + + /** + * Updates the recognition context while the task is running. + * + * @param payloadInput conversation context carried in payload.input + */ + public void updateContext(Map payloadInput) { + if (payloadInput == null) { + throw new ApiException(new InputRequiredException("Parameter invalid: payloadInput context is null")); + } + synchronized (this) { + if (state != RecognitionState.RECOGNITION_STARTED) { + throw new ApiException( + new InputRequiredException( + "State invalid: expect recognition state is started but " + state.getValue())); + } + if (streamDataEmitter == null) { + cmdBuffer.add(AsyncCmdBuffer.builder().streamData(payloadInput).build()); } else { - audioEmitter.onNext(audioFrame); + streamDataEmitter.onNext(payloadInput); } } } @@ -439,10 +462,10 @@ public void stop() { new RuntimeException( "State invalid: expect recognition state is started but " + state.getValue())); } - if (audioEmitter == null) { + if (streamDataEmitter == null) { cmdBuffer.add(AsyncCmdBuffer.builder().isStop(true).build()); } else { - audioEmitter.onComplete(); + streamDataEmitter.onComplete(); } } @@ -455,7 +478,7 @@ public void stop() { } private void reset() { - this.audioEmitter = null; + this.streamDataEmitter = null; this.cmdBuffer.clear(); this.state = RecognitionState.IDLE; this.stopLatch = new AtomicReference<>(null); From 4f68ed1c8b899850094977c32b7dcc2701e5ce00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=81=A5=E4=BB=99?= Date: Wed, 9 Sep 2026 16:50:13 +0800 Subject: [PATCH 2/2] feat(model/omni):support video compactness --- .../dashscope/audio/omni/OmniRealtimeConfig.java | 13 +++++++++++++ .../dashscope/audio/omni/OmniRealtimeConstants.java | 5 +++++ 2 files changed, 18 insertions(+) diff --git a/src/main/java/com/alibaba/dashscope/audio/omni/OmniRealtimeConfig.java b/src/main/java/com/alibaba/dashscope/audio/omni/OmniRealtimeConfig.java index 4f6ae34..504f036 100644 --- a/src/main/java/com/alibaba/dashscope/audio/omni/OmniRealtimeConfig.java +++ b/src/main/java/com/alibaba/dashscope/audio/omni/OmniRealtimeConfig.java @@ -50,6 +50,11 @@ public class OmniRealtimeConfig { * field. */ @Builder.Default OmniRealtimeAudioFormatConfig outputAudio = null; + /** + * Video input representation compactness. When set, it is serialized as {@code + * video.input.representation_compactness}. + */ + @Builder.Default Integer inputVideoRepresentationCompactness = null; /** enable transcription for input audio */ @Builder.Default boolean enableInputAudioTranscription = true; /** model used for input audio transcription */ @@ -106,6 +111,14 @@ public JsonObject getConfig() { config.put(OmniRealtimeConstants.INPUT_AUDIO_FORMAT, inputAudioFormat); config.put(OmniRealtimeConstants.OUTPUT_AUDIO_FORMAT, outputAudioFormat); } + if (inputVideoRepresentationCompactness != null) { + Map videoInput = new HashMap<>(); + videoInput.put( + OmniRealtimeConstants.REPRESENTATION_COMPACTNESS, inputVideoRepresentationCompactness); + Map video = new HashMap<>(); + video.put(OmniRealtimeConstants.VIDEO_INPUT, videoInput); + config.put(OmniRealtimeConstants.VIDEO, video); + } if (enableInputAudioTranscription) { Map inputTranscriptionConfig = new HashMap<>(); inputTranscriptionConfig.put( diff --git a/src/main/java/com/alibaba/dashscope/audio/omni/OmniRealtimeConstants.java b/src/main/java/com/alibaba/dashscope/audio/omni/OmniRealtimeConstants.java index fcd07f1..e8d2fc2 100644 --- a/src/main/java/com/alibaba/dashscope/audio/omni/OmniRealtimeConstants.java +++ b/src/main/java/com/alibaba/dashscope/audio/omni/OmniRealtimeConstants.java @@ -16,6 +16,11 @@ public class OmniRealtimeConstants { public static final String AUDIO_OUTPUT = "output"; public static final String AUDIO_FORMAT = "format"; public static final String AUDIO_FORMAT_TYPE = "type"; + // Video input configuration, see session.update: + // { "video": { "input": { "representation_compactness": 1 } } } + public static final String VIDEO = "video"; + public static final String VIDEO_INPUT = "input"; + public static final String REPRESENTATION_COMPACTNESS = "representation_compactness"; public static final String INPUT_AUDIO_TRANSCRIPTION = "input_audio_transcription"; public static final String INPUT_AUDIO_TRANSCRIPTION_MODEL = "model"; public static final String INPUT_AUDIO_TRANSCRIPTION_CORPUS = "corpus";