Feature/audio add Omni video compactness support and ASR runtime input.context updates via continue-task.

src/main/java/com/alibaba/dashscope/audio/asr/recognition/Recognition.java

@@ -37,12 +37,12 @@ public final class Recognition {
 
   private ApiServiceOption serviceOption;
 
-  private Emitter<ByteBuffer> audioEmitter;
+  private Emitter<Object> streamDataEmitter;
 
   @SuperBuilder
   private static class AsyncCmdBuffer {
     @Builder.Default private boolean isStop = false;
-    private ByteBuffer audioFrame;
+    private Object streamData;
   }
 
   private final Queue<AsyncCmdBuffer> cmdBuffer = new LinkedList<>();
@@ -62,22 +62,22 @@ private static class AsyncCmdBuffer {
   @SuperBuilder
   private static class RecognitionParamWithStream extends RecognitionParam {
 
-    @NonNull private Flowable<ByteBuffer> audioStream;
+    @NonNull private Flowable<Object> streamData;
 
     @Override
     public Flowable<Object> getStreamingData() {
-      return audioStream.cast(Object.class);
+      return streamData;
     }
 
     public static RecognitionParamWithStream FromRecognitionParam(
-        RecognitionParam param, Flowable<ByteBuffer> audioStream, String preRequestId) {
+        RecognitionParam param, Flowable<?> streamData, String preRequestId) {
       RecognitionParamWithStream recognitionParamWithStream =
           RecognitionParamWithStream.builder()
               .parameters((param.getParameters()))
               .parameter("pre_task_id", preRequestId)
               .headers(param.getHeaders())
               .format(param.getFormat())
-              .audioStream(audioStream)
+              .streamData(streamData.cast(Object.class))
               .disfluencyRemovalEnabled(param.isDisfluencyRemovalEnabled())
               .model(param.getModel())
               .sampleRate(param.getSampleRate())
@@ -217,7 +217,7 @@ public void call(RecognitionParam param, ResultCallback<RecognitionResult> callb
           new InputRequiredException("Parameter invalid: ResultCallback is null"));
     }
 
-    Flowable<ByteBuffer> audioFrames =
+    Flowable<Object> audioFrames =
         Flowable.create(
             emitter -> {
               synchronized (Recognition.this) {
@@ -227,12 +227,12 @@ public void call(RecognitionParam param, ResultCallback<RecognitionResult> callb
                       emitter.onComplete();
                       return;
                     } else {
-                      emitter.onNext(buffer.audioFrame);
+                      emitter.onNext(buffer.streamData);
                     }
                   }
                   cmdBuffer.clear();
                 }
-                audioEmitter = emitter;
+                streamDataEmitter = emitter;
               }
             },
             BackpressureStrategy.BUFFER);
@@ -423,10 +423,34 @@ public void sendAudioFrame(ByteBuffer audioFrame) {
             new InputRequiredException(
                 "State invalid: expect recognition state is started but " + state.getValue()));
       }
-      if (audioEmitter == null) {
-        cmdBuffer.add(AsyncCmdBuffer.builder().audioFrame(audioFrame).build());
+      if (streamDataEmitter == null) {
+        cmdBuffer.add(AsyncCmdBuffer.builder().streamData(audioFrame).build());
+      } else {
+        streamDataEmitter.onNext(audioFrame);
+      }
+    }
+  }
+
+  /**
+   * Updates the recognition context while the task is running.
+   *
+   * @param payloadInput conversation context carried in payload.input
+   */
+  public void updateContext(Map<String, Object> payloadInput) {
+    if (payloadInput == null) {
+      throw new ApiException(
+          new InputRequiredException("Parameter invalid: payloadInput context is null"));
+    }
+    synchronized (this) {
+      if (state != RecognitionState.RECOGNITION_STARTED) {
+        throw new ApiException(
+            new InputRequiredException(
+                "State invalid: expect recognition state is started but " + state.getValue()));
+      }
+      if (streamDataEmitter == null) {
+        cmdBuffer.add(AsyncCmdBuffer.builder().streamData(payloadInput).build());
       } else {
-        audioEmitter.onNext(audioFrame);
+        streamDataEmitter.onNext(payloadInput);
       }
     }
   }
@@ -439,10 +463,10 @@ public void stop() {
             new RuntimeException(
                 "State invalid: expect recognition state is started but " + state.getValue()));
       }
-      if (audioEmitter == null) {
+      if (streamDataEmitter == null) {
         cmdBuffer.add(AsyncCmdBuffer.builder().isStop(true).build());
       } else {
-        audioEmitter.onComplete();
+        streamDataEmitter.onComplete();
       }
     }
 
@@ -455,7 +479,7 @@ public void stop() {
   }
 
   private void reset() {
-    this.audioEmitter = null;
+    this.streamDataEmitter = null;
     this.cmdBuffer.clear();
     this.state = RecognitionState.IDLE;
     this.stopLatch = new AtomicReference<>(null);

src/main/java/com/alibaba/dashscope/audio/omni/OmniRealtimeConfig.java

@@ -50,6 +50,11 @@ public class OmniRealtimeConfig {
    * field.
    */
   @Builder.Default OmniRealtimeAudioFormatConfig outputAudio = null;
+  /**
+   * Video input representation compactness. When set, it is serialized as {@code
+   * video.input.representation_compactness}.
+   */
+  @Builder.Default Integer inputVideoRepresentationCompactness = null;
   /** enable transcription for input audio */
   @Builder.Default boolean enableInputAudioTranscription = true;
   /** model used for input audio transcription */
@@ -106,6 +111,14 @@ public JsonObject getConfig() {
       config.put(OmniRealtimeConstants.INPUT_AUDIO_FORMAT, inputAudioFormat);
       config.put(OmniRealtimeConstants.OUTPUT_AUDIO_FORMAT, outputAudioFormat);
     }
+    if (inputVideoRepresentationCompactness != null) {
+      Map<String, Object> videoInput = new HashMap<>();
+      videoInput.put(
+          OmniRealtimeConstants.REPRESENTATION_COMPACTNESS, inputVideoRepresentationCompactness);
+      Map<String, Object> video = new HashMap<>();
+      video.put(OmniRealtimeConstants.VIDEO_INPUT, videoInput);
+      config.put(OmniRealtimeConstants.VIDEO, video);
+    }
     if (enableInputAudioTranscription) {
       Map<String, Object> inputTranscriptionConfig = new HashMap<>();
       inputTranscriptionConfig.put(

src/main/java/com/alibaba/dashscope/audio/omni/OmniRealtimeConstants.java

@@ -16,6 +16,11 @@ public class OmniRealtimeConstants {
   public static final String AUDIO_OUTPUT = "output";
   public static final String AUDIO_FORMAT = "format";
   public static final String AUDIO_FORMAT_TYPE = "type";
+  // Video input configuration, see session.update:
+  // { "video": { "input": { "representation_compactness": 1 } } }
+  public static final String VIDEO = "video";
+  public static final String VIDEO_INPUT = "input";
+  public static final String REPRESENTATION_COMPACTNESS = "representation_compactness";
   public static final String INPUT_AUDIO_TRANSCRIPTION = "input_audio_transcription";
   public static final String INPUT_AUDIO_TRANSCRIPTION_MODEL = "model";
   public static final String INPUT_AUDIO_TRANSCRIPTION_CORPUS = "corpus";