Skip to content

Commit 8ab7f07

Browse files
prasadskarmarkarkrwc
prasadskarmarkar
authored and
krwc
committed
fix: add media/image support in Spring AI MessageConverter
Previously, MessageConverter only transferred text content from ADK to Spring AI, ignoring image and media attachments. This caused vision model requests to fail even though Spring AI's underlying models (like GPT-4o) support image inputs. Updated MessageConverter to properly handle image/media parts by constructing UserMessage with Media attachments. Fixes #705
1 parent 82baba1 commit 8ab7f07

2 files changed

Lines changed: 183 additions & 10 deletions

File tree

‎contrib/spring-ai/src/main/java/com/google/adk/models/springai/MessageConverter.java‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -221,8 +221,7 @@ private List<Message> handleUserContent(Content content) {
221221
} catch (Exception e) {
222222
// Log warning but continue processing other parts
223223
// In production, consider proper logging framework
224-
System.err.println(
225-
"Warning: Failed to parse media mime type: " + blob.mimeType().get());
224+
System.err.println("Warning: Failed to process media part: " + e.getMessage());
226225
}
227226
}
228227
} else if (part.fileData().isPresent()) {
@@ -235,19 +234,14 @@ private List<Message> handleUserContent(Content content) {
235234
URI uri = URI.create(fileData.fileUri().get());
236235
mediaList.add(new Media(mimeType, uri));
237236
} catch (Exception e) {
238-
System.err.println(
239-
"Warning: Failed to parse media mime type: " + fileData.mimeType().get());
237+
System.err.println("Warning: Failed to process media part: " + e.getMessage());
240238
}
241239
}
242240
}
243241
}
244242

245243
List<Message> messages = new ArrayList<>();
246-
// Create UserMessage with text
247-
// TODO: Media attachments support - UserMessage constructors with media are private in Spring
248-
// AI 1.1.0
249-
// For now, only text content is supported
250-
messages.add(new UserMessage(textBuilder.toString()));
244+
messages.add(UserMessage.builder().text(textBuilder.toString()).media(mediaList).build());
251245
messages.addAll(toolResponseMessages);
252246

253247
return messages;

‎contrib/spring-ai/src/test/java/com/google/adk/models/springai/MessageConverterTest.java‎

Lines changed: 180 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,9 @@ void testToLlmPromptWithUserMessage() {
6060
assertThat(prompt.getInstructions()).hasSize(1);
6161
Message message = prompt.getInstructions().get(0);
6262
assertThat(message).isInstanceOf(UserMessage.class);
63-
assertThat(((UserMessage) message).getText()).isEqualTo("Hello, how are you?");
63+
UserMessage userMessage = (UserMessage) message;
64+
assertThat(userMessage.getText()).isEqualTo("Hello, how are you?");
65+
assertThat(userMessage.getMedia()).isEmpty();
6466
}
6567

6668
@Test
@@ -444,4 +446,181 @@ void testCombineMultipleSystemMessagesForGeminiCompatibility() {
444446
assertThat(secondMessage).isInstanceOf(UserMessage.class);
445447
assertThat(((UserMessage) secondMessage).getText()).isEqualTo("Hello world");
446448
}
449+
450+
@Test
451+
void testUserMessageWithInlineMediaData() {
452+
// Test conversion of ADK Content with inline media (image bytes) to Spring AI UserMessage
453+
byte[] imageData = "fake-image-data".getBytes();
454+
String mimeType = "image/png";
455+
456+
Content userContent =
457+
Content.builder()
458+
.role("user")
459+
.parts(
460+
List.of(
461+
Part.fromText("What's in this image?"),
462+
Part.builder()
463+
.inlineData(
464+
com.google.genai.types.Blob.builder()
465+
.mimeType(mimeType)
466+
.data(imageData)
467+
.build())
468+
.build()))
469+
.build();
470+
471+
LlmRequest request = LlmRequest.builder().contents(List.of(userContent)).build();
472+
473+
Prompt prompt = messageConverter.toLlmPrompt(request);
474+
475+
assertThat(prompt.getInstructions()).hasSize(1);
476+
Message message = prompt.getInstructions().get(0);
477+
assertThat(message).isInstanceOf(UserMessage.class);
478+
479+
UserMessage userMessage = (UserMessage) message;
480+
assertThat(userMessage.getText()).isEqualTo("What's in this image?");
481+
assertThat(userMessage.getMedia()).hasSize(1);
482+
org.springframework.ai.content.Media media = userMessage.getMedia().get(0);
483+
assertThat(media.getMimeType().toString()).isEqualTo(mimeType);
484+
assertThat(media.getData()).isInstanceOf(byte[].class);
485+
byte[] actualData = (byte[]) media.getData();
486+
assertThat(actualData).isEqualTo(imageData);
487+
}
488+
489+
@Test
490+
void testUserMessageWithFileMediaData() {
491+
// Test conversion of ADK Content with file-based media (URI) to Spring AI UserMessage
492+
String fileUri = "gs://bucket/image.jpg";
493+
String mimeType = "image/jpeg";
494+
495+
Content userContent =
496+
Content.builder()
497+
.role("user")
498+
.parts(
499+
List.of(
500+
Part.fromText("Analyze this image"),
501+
Part.builder()
502+
.fileData(
503+
com.google.genai.types.FileData.builder()
504+
.mimeType(mimeType)
505+
.fileUri(fileUri)
506+
.build())
507+
.build()))
508+
.build();
509+
510+
LlmRequest request = LlmRequest.builder().contents(List.of(userContent)).build();
511+
512+
Prompt prompt = messageConverter.toLlmPrompt(request);
513+
514+
assertThat(prompt.getInstructions()).hasSize(1);
515+
Message message = prompt.getInstructions().get(0);
516+
assertThat(message).isInstanceOf(UserMessage.class);
517+
518+
UserMessage userMessage = (UserMessage) message;
519+
assertThat(userMessage.getText()).isEqualTo("Analyze this image");
520+
assertThat(userMessage.getMedia()).hasSize(1);
521+
org.springframework.ai.content.Media media = userMessage.getMedia().get(0);
522+
assertThat(media.getMimeType().toString()).isEqualTo(mimeType);
523+
assertThat(media.getData()).isInstanceOf(String.class);
524+
String actualUri = (String) media.getData();
525+
assertThat(actualUri).isEqualTo(fileUri);
526+
}
527+
528+
@Test
529+
void testUserMessageWithMultipleMediaAttachments() {
530+
// Test conversion with multiple media attachments
531+
byte[] image1 = "image1-data".getBytes();
532+
byte[] image2 = "image2-data".getBytes();
533+
534+
Content userContent =
535+
Content.builder()
536+
.role("user")
537+
.parts(
538+
List.of(
539+
Part.fromText("Compare these images"),
540+
Part.builder()
541+
.inlineData(
542+
com.google.genai.types.Blob.builder()
543+
.mimeType("image/png")
544+
.data(image1)
545+
.build())
546+
.build(),
547+
Part.builder()
548+
.inlineData(
549+
com.google.genai.types.Blob.builder()
550+
.mimeType("image/jpeg")
551+
.data(image2)
552+
.build())
553+
.build()))
554+
.build();
555+
556+
LlmRequest request = LlmRequest.builder().contents(List.of(userContent)).build();
557+
558+
Prompt prompt = messageConverter.toLlmPrompt(request);
559+
560+
assertThat(prompt.getInstructions()).hasSize(1);
561+
UserMessage userMessage = (UserMessage) prompt.getInstructions().get(0);
562+
assertThat(userMessage.getText()).isEqualTo("Compare these images");
563+
assertThat(userMessage.getMedia()).hasSize(2);
564+
}
565+
566+
@Test
567+
void testUserMessageWithInvalidMimeTypeGracefullySkipsMediaPart() {
568+
// Test that an invalid MIME type string causes the media part to be skipped gracefully
569+
byte[] imageData = "fake-image-data".getBytes();
570+
571+
Content userContent =
572+
Content.builder()
573+
.role("user")
574+
.parts(
575+
List.of(
576+
Part.fromText("What's in this image?"),
577+
Part.builder()
578+
.inlineData(
579+
com.google.genai.types.Blob.builder()
580+
.mimeType("invalid/mime/type!!!") // invalid MIME type
581+
.data(imageData)
582+
.build())
583+
.build()))
584+
.build();
585+
586+
LlmRequest request = LlmRequest.builder().contents(List.of(userContent)).build();
587+
588+
// Should not throw — invalid MIME type is silently skipped
589+
Prompt prompt = messageConverter.toLlmPrompt(request);
590+
591+
assertThat(prompt.getInstructions()).hasSize(1);
592+
UserMessage userMessage = (UserMessage) prompt.getInstructions().get(0);
593+
assertThat(userMessage.getText()).isEqualTo("What's in this image?");
594+
// Media part is skipped due to invalid MIME type
595+
assertThat(userMessage.getMedia()).isEmpty();
596+
}
597+
598+
@Test
599+
void testUserMessageWithMediaOnly() {
600+
// Test conversion with media but no text
601+
byte[] imageData = "image-only".getBytes();
602+
603+
Content userContent =
604+
Content.builder()
605+
.role("user")
606+
.parts(
607+
List.of(
608+
Part.builder()
609+
.inlineData(
610+
com.google.genai.types.Blob.builder()
611+
.mimeType("image/png")
612+
.data(imageData)
613+
.build())
614+
.build()))
615+
.build();
616+
617+
LlmRequest request = LlmRequest.builder().contents(List.of(userContent)).build();
618+
619+
Prompt prompt = messageConverter.toLlmPrompt(request);
620+
621+
assertThat(prompt.getInstructions()).hasSize(1);
622+
UserMessage userMessage = (UserMessage) prompt.getInstructions().get(0);
623+
assertThat(userMessage.getText()).isEmpty();
624+
assertThat(userMessage.getMedia()).hasSize(1);
625+
}
447626
}

0 commit comments

Comments
 (0)