diff --git a/api/src/main/java/com/theokanning/openai/completion/chat/ImageContent.java b/api/src/main/java/com/theokanning/openai/completion/chat/ImageContent.java index b86e2d6..3088366 100644 --- a/api/src/main/java/com/theokanning/openai/completion/chat/ImageContent.java +++ b/api/src/main/java/com/theokanning/openai/completion/chat/ImageContent.java @@ -16,10 +16,12 @@ /** * @author LiangTao * @date 2024年04月10 10:26 + * @deprecated use {@link MultiMediaContent},the name is not accurate,use new class {@link MultiMediaContent} instead and the refactoring is done **/ @Data @AllArgsConstructor @NoArgsConstructor +@Deprecated public class ImageContent { /** diff --git a/api/src/main/java/com/theokanning/openai/completion/chat/ImageUrl.java b/api/src/main/java/com/theokanning/openai/completion/chat/ImageUrl.java index 845d9e6..dd2a2ca 100644 --- a/api/src/main/java/com/theokanning/openai/completion/chat/ImageUrl.java +++ b/api/src/main/java/com/theokanning/openai/completion/chat/ImageUrl.java @@ -21,6 +21,4 @@ public class ImageUrl { public ImageUrl(String url) { this.url = url; } - - } diff --git a/api/src/main/java/com/theokanning/openai/completion/chat/MultiMediaContent.java b/api/src/main/java/com/theokanning/openai/completion/chat/MultiMediaContent.java new file mode 100644 index 0000000..521a54a --- /dev/null +++ b/api/src/main/java/com/theokanning/openai/completion/chat/MultiMediaContent.java @@ -0,0 +1,120 @@ +package com.theokanning.openai.completion.chat; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.theokanning.openai.assistants.message.content.ImageFile; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Base64; + +/** + * @author LiangTao + * @date 2024年04月10 10:26 + **/ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class MultiMediaContent { + + /** + * The type of the content. Either "text", "image_url" or "input_audio". + */ + @NonNull + private String type; + + @JsonInclude(JsonInclude.Include.NON_NULL) + private String text; + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty("image_url") + private ImageUrl imageUrl; + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty("image_file") + private ImageFile imageFile; + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty("input_audio") + private InputAudio inputAudio; + + public MultiMediaContent(String text) { + this.type = "text"; + this.text = text; + } + + public MultiMediaContent(ImageUrl imageUrl) { + this.type = "image_url"; + this.imageUrl = imageUrl; + } + + public MultiMediaContent(InputAudio inputAudio) { + this.type = "input_audio"; + this.inputAudio = inputAudio; + } + + /** + * build an image content from a file path, detail is auto + * @author liangtao + * @date 2024/11/28 + * @param imagePath + * @return com.theokanning.openai.completion.chat.ImageContent + **/ + public static MultiMediaContent ofImagePath(Path imagePath){ + return ofImagePath(imagePath,"auto"); + } + + /** + * build an image content from a file path,Specify detail + * @author liangtao + * @date 2024/11/28 + * @param imagePath + * @param detail level: "auto", "low", "high" + * @return com.theokanning.openai.completion.chat.ImageContent + **/ + public static MultiMediaContent ofImagePath(Path imagePath, String detail){ + String imagePathString = imagePath.toAbsolutePath().toString(); + String extension = imagePathString.substring(imagePathString.lastIndexOf('.') + 1); + ImageUrl imageUrl = new ImageUrl("data:image/" + extension + ";base64," + encode2base64(imagePath),detail); + return new MultiMediaContent(imageUrl); + } + + /** + * build an image content from a url + * @param imageUrl url + * @param detail level: "auto", "low", "high" + */ + public static MultiMediaContent ofImageUrl(String imageUrl, String detail) { + return new MultiMediaContent(new ImageUrl(imageUrl, detail)); + } + + public static MultiMediaContent ofImageUrl(String imageUrl) { + return ofImageUrl(imageUrl, "auto"); + } + + + + public static MultiMediaContent ofAudioPath(Path inputAudioPath) { + String inputAudioPathString = inputAudioPath.toAbsolutePath().toString(); + String extension = inputAudioPathString.substring(inputAudioPathString.lastIndexOf('.') + 1); + String base64 = encode2base64(inputAudioPath); + InputAudio inputAudio = new InputAudio(base64, extension); + return new MultiMediaContent(inputAudio); + } + + + private static String encode2base64(Path path) { + byte[] fileContent; + try { + fileContent = Files.readAllBytes(path); + return Base64.getEncoder().encodeToString(fileContent); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/api/src/main/java/com/theokanning/openai/completion/chat/UserMessage.java b/api/src/main/java/com/theokanning/openai/completion/chat/UserMessage.java index 48dd720..8b50f09 100644 --- a/api/src/main/java/com/theokanning/openai/completion/chat/UserMessage.java +++ b/api/src/main/java/com/theokanning/openai/completion/chat/UserMessage.java @@ -7,10 +7,11 @@ import lombok.Data; import lombok.NoArgsConstructor; -import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; -import java.util.*; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; /** @@ -34,13 +35,21 @@ public UserMessage(String content) { this.content = content; } + public static UserMessageBuilder builder() { + return new UserMessageBuilder(); + } + + /** + * + * @param imageContents + */ + @Deprecated public UserMessage(List imageContents) { this.content = imageContents; } - @Override @JsonIgnore public String getTextContent() { @@ -61,13 +70,14 @@ public String getTextContent() { /** * 构件一个图片识别请求消息,支持多个图片 - * + * @deprecated use {@link UserMessageBuilder#buildImageMessage(String, String...)} instead * @param prompt query text * @param imageUrls image urls * @return com.theokanning.openai.completion.chat.UserMessage * @author liangtao * @date 2024/4/12 **/ + @Deprecated public static UserMessage buildImageMessage(String prompt, String... imageUrls) { List imageContents = Arrays.stream(imageUrls).map(url -> new ImageContent(new ImageUrl(url))).collect(Collectors.toList()); imageContents.add(0, new ImageContent(prompt)); @@ -75,31 +85,41 @@ public static UserMessage buildImageMessage(String prompt, String... imageUrls) } /** - * 构件一个图片识别请求消息,支持多个图片 + * 构件一个图片识别请求消息,支持多个图片,detail默认为auto + * @deprecated use {@link UserMessageBuilder#buildImageMessage(String, String...)} instead * @author liangtao * @date 2024/8/15 * @param prompt query text * @param imagePaths 文件本地路径 * @return com.theokanning.openai.completion.chat.UserMessage **/ + @Deprecated public static UserMessage buildImageMessage(String prompt, Path... imagePaths) { List imageContents = Arrays.stream(imagePaths).map(ImageContent::ofImagePath).collect(Collectors.toList()); imageContents.add(0, new ImageContent(prompt)); return new UserMessage(imageContents); } + + /** * 构建一个音频识别请求消息,支持多个音频 + * @deprecated use {@link UserMessageBuilder#buildAudioMessage(String, Path...)} instead * @param prompt query text * @param inputAudioPaths 音频文件本地路径 * @return com.theokanning.openai.completion.chat.UserMessage * @author Allen Hu * @date 2024/11/6 */ + @Deprecated public static UserMessage buildInputAudioMessage(String prompt, Path... inputAudioPaths) { List imageContents = Arrays.stream(inputAudioPaths).map(ImageContent::ofAudioPath).collect(Collectors.toList()); imageContents.add(0, new ImageContent(prompt)); return new UserMessage(imageContents); } + + public static void main(String[] args) { + // UserMessage.builder().withMultiMediaContent() + } } diff --git a/api/src/main/java/com/theokanning/openai/completion/chat/UserMessageBuilder.java b/api/src/main/java/com/theokanning/openai/completion/chat/UserMessageBuilder.java new file mode 100644 index 0000000..c2ac1a9 --- /dev/null +++ b/api/src/main/java/com/theokanning/openai/completion/chat/UserMessageBuilder.java @@ -0,0 +1,143 @@ +package com.theokanning.openai.completion.chat; + +import lombok.extern.slf4j.Slf4j; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author LiangTao + * @date 2024年11月29 15:18 + **/ +@Slf4j +public class UserMessageBuilder { + + private final UserMessage userMessage; + + public UserMessageBuilder() { + userMessage = new UserMessage(); + } + + public UserMessageBuilder withName(String name) { + userMessage.setName(name); + return this; + } + + /** + * Set the text content of the message. + * @param text the text content of the message + */ + public UserMessageBuilder withTextMessage(String text) { + userMessage.setContent(text); + return this; + } + + + /** + * build the message with image/audio content + * @param multiMediaContent the image content or audio content of the message + * @return the UserMessageBuilder + */ + public UserMessageBuilder withMultiMediaContent(List multiMediaContent) { + userMessage.setContent(multiMediaContent); + return this; + } + + + /** + * This method supports adding image/audio message content to assist the component UserMessage + * @param multiMediaContents image audio message content + */ + public UserMessageBuilder addMultiMediaContents(MultiMediaContent... multiMediaContents) { + Object content = userMessage.getContent(); + if (content==null){ + ArrayList multiMediaContentList = new ArrayList<>(); + userMessage.setContent(multiMediaContentList); + }else { + if (content instanceof String){ + log.error("The content of the message is text, can not add image content"); + return this; + } + } + List multiMediaContentList = (List) content; + multiMediaContentList.addAll(Arrays.asList(multiMediaContents)); + return this; + } + + /** + * Used for component image messages, supporting multiple images + * @param prompt prompt + * @param imageUrl image url or base64 image data + */ + public UserMessage buildImageMessage(String prompt, String imageUrl) { + return buildImageMessageWithDetail(prompt,"auto",imageUrl); + } + + /** + * Used for component image messages, supporting multiple images + * @param prompt prompt + * @param detail level: "auto", "low", "high" + * @param imageUrls image urls or base64 image data + */ + public UserMessage buildImageMessageWithDetail(String prompt, String detail,String... imageUrls) { + List imageContents = Arrays.stream(imageUrls).map(url -> MultiMediaContent.ofImageUrl(url,detail)).collect(Collectors.toList()); + imageContents.add(0, new MultiMediaContent(prompt)); + userMessage.setContent(imageContents); + return build(); + } + + /** + * Used for component image messages, supporting multiple images + * @param prompt prompt + * @param imageUrls image urls or base64 image data + */ + public UserMessage buildImageMessage(String prompt, String... imageUrls) { + return buildImageMessageWithDetail(prompt,"auto",imageUrls); + } + + /** + * Used for component image messages, supporting multiple images + * @param prompt prompt + * @param detail level: "auto", "low", "high" + * @param imagePaths image file paths + */ + public UserMessage buildImageMessageWithDetail(String prompt, String detail, Path... imagePaths) { + List imageContents = Arrays.stream(imagePaths).map(path -> MultiMediaContent.ofImagePath(path,detail)).collect(Collectors.toList()); + imageContents.add(0, new MultiMediaContent(prompt)); + userMessage.setContent(imageContents); + return build(); + } + + + /** + * Used for component image messages, supporting multiple images + * @param prompt prompt + * @param imagePaths image file paths + */ + public UserMessage buildImageMessage(String prompt, Path... imagePaths) { + return buildImageMessageWithDetail(prompt,"auto",imagePaths); + } + + /** + * Used for component audio messages, supporting multiple audio + * @param prompt prompt + * @param audioPaths audio file paths + */ + public UserMessage buildAudioMessage(String prompt, Path... audioPaths) { + List audioContents = Arrays.stream(audioPaths).map(MultiMediaContent::ofAudioPath).collect(Collectors.toList()); + audioContents.add(0, new MultiMediaContent(prompt)); + userMessage.setContent(audioContents); + return build(); + } + + + + public UserMessage build() { + return userMessage; + } + + +}