Lambdua · Lambdua · Nov 29, 2024 · Nov 29, 2024
diff --git a/api/src/main/java/com/theokanning/openai/completion/chat/ImageContent.java b/api/src/main/java/com/theokanning/openai/completion/chat/ImageContent.java
@@ -16,10 +16,12 @@
 /**
  * @author LiangTao
  * @date 2024年04月10 10:26
+ * @deprecated use {@link MultiMediaContent},the name is not accurate,use new class {@link MultiMediaContent} instead and the refactoring is done
  **/
 @Data
 @AllArgsConstructor
 @NoArgsConstructor
+@Deprecated
 public class ImageContent {
 
     /**

diff --git a/api/src/main/java/com/theokanning/openai/completion/chat/ImageUrl.java b/api/src/main/java/com/theokanning/openai/completion/chat/ImageUrl.java
@@ -21,6 +21,4 @@ public class ImageUrl {
     public ImageUrl(String url) {
         this.url = url;
     }
-
-
 }
diff --git a/api/src/main/java/com/theokanning/openai/completion/chat/MultiMediaContent.java b/api/src/main/java/com/theokanning/openai/completion/chat/MultiMediaContent.java
@@ -0,0 +1,120 @@
+package com.theokanning.openai.completion.chat;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.theokanning.openai.assistants.message.content.ImageFile;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Base64;
+
+/**
+ * @author LiangTao
+ * @date 2024年04月10 10:26
+ **/
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class MultiMediaContent {
+
+    /**
+     * The type of the content. Either "text", "image_url" or "input_audio".
+     */
+    @NonNull
+    private String type;
+
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    private String text;
+
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    @JsonProperty("image_url")
+    private ImageUrl imageUrl;
+
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    @JsonProperty("image_file")
+    private ImageFile imageFile;
+
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    @JsonProperty("input_audio")
+    private InputAudio inputAudio;
+
+    public MultiMediaContent(String text) {
+        this.type = "text";
+        this.text = text;
+    }
+
+    public MultiMediaContent(ImageUrl imageUrl) {
+        this.type = "image_url";
+        this.imageUrl = imageUrl;
+    }
+
+    public MultiMediaContent(InputAudio inputAudio) {
+        this.type = "input_audio";
+        this.inputAudio = inputAudio;
+    }
+
+    /**
+     * build an image content from a file path, detail is auto
+     * @author liangtao
+     * @date 2024/11/28
+     * @param imagePath
+     * @return com.theokanning.openai.completion.chat.ImageContent
+     **/
+    public static MultiMediaContent ofImagePath(Path imagePath){
+        return ofImagePath(imagePath,"auto");
+    }
+
+    /**
+     * build an image content from a file path,Specify detail
+     * @author liangtao
+     * @date 2024/11/28
+     * @param imagePath
+     * @param detail level: "auto", "low", "high"
+     * @return com.theokanning.openai.completion.chat.ImageContent
+     **/
+    public static MultiMediaContent ofImagePath(Path imagePath, String detail){
+        String imagePathString = imagePath.toAbsolutePath().toString();
+        String extension = imagePathString.substring(imagePathString.lastIndexOf('.') + 1);
+        ImageUrl imageUrl = new ImageUrl("data:image/" + extension + ";base64," + encode2base64(imagePath),detail);
+        return new MultiMediaContent(imageUrl);
+    }
+
+    /**
+     * build an image content from a url
+     * @param imageUrl url
+     * @param detail level: "auto", "low", "high"
+     */
+    public static MultiMediaContent ofImageUrl(String imageUrl, String detail) {
+        return new MultiMediaContent(new ImageUrl(imageUrl, detail));
+    }
+
+    public static MultiMediaContent ofImageUrl(String imageUrl) {
+        return ofImageUrl(imageUrl, "auto");
+    }
+
+
+
+    public static MultiMediaContent ofAudioPath(Path inputAudioPath) {
+        String inputAudioPathString = inputAudioPath.toAbsolutePath().toString();
+        String extension = inputAudioPathString.substring(inputAudioPathString.lastIndexOf('.') + 1);
+        String base64 = encode2base64(inputAudioPath);
+        InputAudio inputAudio = new InputAudio(base64, extension);
+        return new MultiMediaContent(inputAudio);
+    }
+
+
+    private static String encode2base64(Path path) {
+        byte[] fileContent;
+        try {
+            fileContent = Files.readAllBytes(path);
+            return Base64.getEncoder().encodeToString(fileContent);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/api/src/main/java/com/theokanning/openai/completion/chat/UserMessage.java b/api/src/main/java/com/theokanning/openai/completion/chat/UserMessage.java
@@ -7,10 +7,11 @@
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
-import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.*;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 /**
@@ -34,13 +35,21 @@ public UserMessage(String content) {
         this.content = content;
     }
 
+    public static UserMessageBuilder builder() {
+        return new UserMessageBuilder();
+    }
+
+    /**
+     *
+     * @param imageContents
+     */
+    @Deprecated
     public UserMessage(List<ImageContent> imageContents) {
         this.content = imageContents;
     }
 
 
 
-
     @Override
     @JsonIgnore
     public String getTextContent() {
@@ -61,45 +70,56 @@ public String getTextContent() {
 
     /**
      * 构件一个图片识别请求消息,支持多个图片
-     *
+     * @deprecated use {@link UserMessageBuilder#buildImageMessage(String, String...)} instead
      * @param prompt      query text
      * @param imageUrls image urls
      * @return com.theokanning.openai.completion.chat.UserMessage
      * @author liangtao
      * @date 2024/4/12
      **/
+    @Deprecated
     public static UserMessage buildImageMessage(String prompt, String... imageUrls) {
         List<ImageContent> imageContents = Arrays.stream(imageUrls).map(url -> new ImageContent(new ImageUrl(url))).collect(Collectors.toList());
         imageContents.add(0, new ImageContent(prompt));
         return new UserMessage(imageContents);
     }
 
     /**
-     * 构件一个图片识别请求消息,支持多个图片
+     * 构件一个图片识别请求消息,支持多个图片,detail默认为auto
+     * @deprecated use {@link UserMessageBuilder#buildImageMessage(String, String...)} instead
      * @author liangtao
      * @date 2024/8/15
      * @param prompt query text
      * @param imagePaths 文件本地路径
      * @return com.theokanning.openai.completion.chat.UserMessage
      **/
+    @Deprecated
     public  static UserMessage buildImageMessage(String prompt, Path... imagePaths) {
         List<ImageContent> imageContents = Arrays.stream(imagePaths).map(ImageContent::ofImagePath).collect(Collectors.toList());
         imageContents.add(0, new ImageContent(prompt));
         return new UserMessage(imageContents);
     }
 
+
+
     /**
      * 构建一个音频识别请求消息,支持多个音频
+     * @deprecated use {@link UserMessageBuilder#buildAudioMessage(String, Path...)} instead
      * @param prompt query text
      * @param inputAudioPaths 音频文件本地路径
      * @return com.theokanning.openai.completion.chat.UserMessage
      * @author Allen Hu
      * @date 2024/11/6
      */
+    @Deprecated
     public static UserMessage buildInputAudioMessage(String prompt, Path... inputAudioPaths) {
         List<ImageContent> imageContents = Arrays.stream(inputAudioPaths).map(ImageContent::ofAudioPath).collect(Collectors.toList());
         imageContents.add(0, new ImageContent(prompt));
         return new UserMessage(imageContents);
     }
+
+    public static void main(String[] args) {
+        // UserMessage.builder().withMultiMediaContent()
+    }
 }
 
diff --git a/api/src/main/java/com/theokanning/openai/completion/chat/UserMessageBuilder.java b/api/src/main/java/com/theokanning/openai/completion/chat/UserMessageBuilder.java
@@ -0,0 +1,143 @@
+package com.theokanning.openai.completion.chat;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @author LiangTao
+ * @date 2024年11月29 15:18
+ **/
+@Slf4j
+public class UserMessageBuilder {
+
+    private final UserMessage userMessage;
+
+    public UserMessageBuilder() {
+        userMessage = new UserMessage();
+    }
+
+    public UserMessageBuilder withName(String name) {
+        userMessage.setName(name);
+        return this;
+    }
+
+    /**
+     * Set the text content of the message.
+     * @param text the text content of the message
+     */
+    public UserMessageBuilder withTextMessage(String text) {
+        userMessage.setContent(text);
+        return this;
+    }
+
+
+    /**
+     * build the message with image/audio  content
+     * @param multiMediaContent the image content or audio content  of the message
+     * @return the UserMessageBuilder
+     */
+    public UserMessageBuilder withMultiMediaContent(List<MultiMediaContent> multiMediaContent) {
+        userMessage.setContent(multiMediaContent);
+        return this;
+    }
+
+
+    /**
+     * This method supports adding image/audio message content to assist the component UserMessage
+     * @param multiMediaContents image audio message content
+     */
+    public UserMessageBuilder addMultiMediaContents(MultiMediaContent... multiMediaContents) {
+        Object content = userMessage.getContent();
+        if (content==null){
+            ArrayList<MultiMediaContent> multiMediaContentList = new ArrayList<>();
+            userMessage.setContent(multiMediaContentList);
+        }else {
+            if (content instanceof String){
+                log.error("The content of the message is text, can not add image content");
+                return this;
+            }
+        }
+        List<MultiMediaContent> multiMediaContentList = (List<MultiMediaContent>) content;
+        multiMediaContentList.addAll(Arrays.asList(multiMediaContents));
+        return this;
+    }
+
+    /**
+     * Used for component image messages, supporting multiple images
+     * @param prompt  prompt
+     * @param imageUrl image url or base64 image data
+     */
+    public UserMessage buildImageMessage(String prompt, String imageUrl) {
+        return buildImageMessageWithDetail(prompt,"auto",imageUrl);
+    }
+
+    /**
+     * Used for component image messages, supporting multiple images
+     * @param prompt  prompt
+     * @param detail level: "auto", "low", "high"
+     * @param imageUrls image urls or base64 image data
+     */
+    public UserMessage buildImageMessageWithDetail(String prompt, String detail,String... imageUrls) {
+        List<MultiMediaContent> imageContents = Arrays.stream(imageUrls).map(url -> MultiMediaContent.ofImageUrl(url,detail)).collect(Collectors.toList());
+        imageContents.add(0, new MultiMediaContent(prompt));
+        userMessage.setContent(imageContents);
+        return build();
+    }
+
+    /**
+     * Used for component image messages, supporting multiple images
+     * @param prompt  prompt
+     * @param imageUrls image urls or base64 image data
+     */
+    public UserMessage buildImageMessage(String prompt, String... imageUrls) {
+        return buildImageMessageWithDetail(prompt,"auto",imageUrls);
+    }
+
+    /**
+     * Used for component image messages, supporting multiple images
+     * @param prompt prompt
+     * @param detail level: "auto", "low", "high"
+     * @param imagePaths image file paths
+     */
+    public UserMessage buildImageMessageWithDetail(String prompt, String detail, Path... imagePaths) {
+        List<MultiMediaContent> imageContents = Arrays.stream(imagePaths).map(path -> MultiMediaContent.ofImagePath(path,detail)).collect(Collectors.toList());
+        imageContents.add(0, new MultiMediaContent(prompt));
+        userMessage.setContent(imageContents);
+        return build();
+    }
+
+
+    /**
+     * Used for component image messages, supporting multiple images
+     * @param prompt prompt
+     * @param imagePaths image file paths
+     */
+    public UserMessage buildImageMessage(String prompt, Path... imagePaths) {
+        return buildImageMessageWithDetail(prompt,"auto",imagePaths);
+    }
+
+    /**
+     * Used for component audio messages, supporting multiple audio
+     * @param prompt prompt
+     * @param audioPaths audio file paths
+     */
+    public UserMessage buildAudioMessage(String prompt, Path... audioPaths) {
+        List<MultiMediaContent> audioContents = Arrays.stream(audioPaths).map(MultiMediaContent::ofAudioPath).collect(Collectors.toList());
+        audioContents.add(0, new MultiMediaContent(prompt));
+        userMessage.setContent(audioContents);
+        return build();
+    }
+
+
+
+    public UserMessage build() {
+        return userMessage;
+    }
+
+
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -21,6 +21,4 @@ public class ImageUrl { @@
         public ImageUrl(String url) {
             this.url = url;
         }
     }