Add support for audio chat completion (#54)

closes #53

Author: sergioverde90

src/main/java/dev/ai4j/openai4j/chat/Content.java

@@ -21,11 +21,14 @@ public final class Content {
     private final String text;
     @JsonProperty
     private final ImageUrl imageUrl;
+    @JsonProperty
+    private final InputAudio inputAudio;
 
     public Content(Builder builder) {
         this.type = builder.type;
         this.text = builder.text;
         this.imageUrl = builder.imageUrl;
+        this.inputAudio = builder.inputAudio;
     }
 
     public ContentType type() {
@@ -40,6 +43,10 @@ public ImageUrl imageUrl() {
         return imageUrl;
     }
 
+    public InputAudio inputAudio() {
+        return inputAudio;
+    }
+
     @Override
     public boolean equals(Object another) {
         if (this == another) return true;
@@ -50,7 +57,8 @@ public boolean equals(Object another) {
     private boolean equalTo(Content another) {
         return Objects.equals(type, another.type)
                 && Objects.equals(text, another.text)
-                && Objects.equals(imageUrl, another.imageUrl);
+                && Objects.equals(imageUrl, another.imageUrl)
+                && Objects.equals(inputAudio, another.inputAudio);
     }
 
     @Override
@@ -59,6 +67,7 @@ public int hashCode() {
         h += (h << 5) + Objects.hashCode(type);
         h += (h << 5) + Objects.hashCode(text);
         h += (h << 5) + Objects.hashCode(imageUrl);
+        h += (h << 5) + Objects.hashCode(inputAudio);
         return h;
     }
 
@@ -68,6 +77,7 @@ public String toString() {
                 "type=" + type +
                 ", text=" + text +
                 ", imageUrl=" + imageUrl +
+                ", inputAudio=" + inputAudio +
                 "}";
     }
 
@@ -83,6 +93,7 @@ public static final class Builder {
         private ContentType type;
         private String text;
         private ImageUrl imageUrl;
+        private InputAudio inputAudio;
 
         public Builder type(ContentType type) {
             this.type = type;
@@ -99,6 +110,11 @@ public Builder imageUrl(ImageUrl imageUrl) {
             return this;
         }
 
+        public Builder inputAudio(InputAudio inputAudio) {
+            this.inputAudio = inputAudio;
+            return this;
+        }
+
         public Content build() {
             return new Content(this);
         }

src/main/java/dev/ai4j/openai4j/chat/ContentType.java

@@ -7,5 +7,7 @@ public enum ContentType {
     @JsonProperty("text")
     TEXT,
     @JsonProperty("image_url")
-    IMAGE_URL
+    IMAGE_URL,
+    @JsonProperty("input_audio")
+    AUDIO
 }

src/main/java/dev/ai4j/openai4j/chat/InputAudio.java

@@ -0,0 +1,87 @@
+package dev.ai4j.openai4j.chat;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.PropertyNamingStrategies;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonNaming;
+
+import java.util.Objects;
+
+@JsonDeserialize(builder = InputAudio.Builder.class)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
+public class InputAudio {
+
+    private final String data;
+    private final String format;
+
+    private InputAudio(Builder builder) {
+        data = builder.data;
+        format = builder.format;
+    }
+
+    public String getData() {
+        return data;
+    }
+
+    public String getFormat() {
+        return format;
+    }
+
+    @Override
+    public boolean equals(Object another) {
+        if (this == another) return true;
+        return another instanceof InputAudio
+                && equalTo((InputAudio) another);
+    }
+
+    private boolean equalTo(InputAudio another) {
+        return Objects.equals(data, another.data)
+                && Objects.equals(format, another.format);
+    }
+
+    @Override
+    public int hashCode() {
+        int h = 5381;
+        h += (h << 5) + Objects.hashCode(data);
+        h += (h << 5) + Objects.hashCode(format);
+        return h;
+    }
+
+    @Override
+    public String toString() {
+        return "InputAudio{" +
+                "data=" + data +
+                ", format=" + format +
+                "}";
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+    
+    public static final class Builder {
+        
+        private String data;
+        private String format;
+
+        public Builder data(String data) {
+            this.data = data;
+            return this;
+        }
+
+        public Builder format(String format) {
+            this.format = format;
+            return this;
+        }
+
+        public static Builder builder() {
+            return new Builder();
+        }
+
+        public InputAudio build() {
+            return new InputAudio(this);
+        }
+        
+    }
+}

src/main/java/dev/ai4j/openai4j/chat/UserMessage.java

@@ -107,9 +107,7 @@ private Builder() {
         }
 
         public Builder addText(String text) {
-            if (this.content == null) {
-                this.content = new ArrayList<>();
-            }
+            initializeContent();
             Content content = Content.builder()
                     .type(TEXT)
                     .text(text)
@@ -123,9 +121,7 @@ public Builder addImageUrl(String imageUrl) {
         }
 
         public Builder addImageUrl(String imageUrl, ImageDetail imageDetail) {
-            if (this.content == null) {
-                this.content = new ArrayList<>();
-            }
+            initializeContent();
             Content content = Content.builder()
                     .type(IMAGE_URL)
                     .imageUrl(ImageUrl.builder()
@@ -143,6 +139,18 @@ public Builder addImageUrls(String... imageUrls) {
             }
             return this;
         }
+        
+        public Builder addInputAudio(InputAudio inputAudio) {
+            initializeContent();
+            this.content.add(
+                Content.builder()
+                    .type(ContentType.AUDIO)
+                    .inputAudio(inputAudio)
+                .build()
+            );
+            
+            return this;
+        }
 
         public Builder content(List<Content> content) {
             if (content != null) {
@@ -164,5 +172,11 @@ public Builder name(String name) {
         public UserMessage build() {
             return new UserMessage(this);
         }
+
+        private void initializeContent() {
+            if (this.content == null) {
+                this.content = new ArrayList<>();
+            }
+        }
     }
 }

src/test/java/dev/ai4j/openai4j/chat/ChatCompletionAsyncTest.java

@@ -6,6 +6,9 @@
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.EnumSource;
 
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 
@@ -479,4 +482,38 @@ void testGpt4Vision() throws Exception {
         // then
         assertThat(response.content()).containsIgnoringCase("cat");
     }
+
+    @Test
+    void testGpt4Audio() throws Exception {
+
+        // given
+        URL resource = getClass().getClassLoader().getResource("sample.b64");
+        final byte[] bytes = Files.readAllBytes(Paths.get(resource.toURI()));
+
+        ChatCompletionRequest request = ChatCompletionRequest.builder()
+                .model("gpt-4o-audio-preview")
+                .messages(UserMessage.builder()
+                        .addText("What is on the audio?")
+                        .addInputAudio(InputAudio.builder()
+                                .format("wav")
+                                .data(new String(bytes))
+                                .build())
+                        .build())
+                .maxCompletionTokens(100)
+                .temperature(0.0)
+                .build();
+
+        CompletableFuture<ChatCompletionResponse> future = new CompletableFuture<>();
+
+        // when
+        client.chatCompletion(request)
+                .onResponse(future::complete)
+                .onError(future::completeExceptionally)
+                .execute();
+
+        ChatCompletionResponse response = future.get(30, SECONDS);
+
+        // then
+        assertThat(response.content()).containsIgnoringCase("hello");
+    }
 }