Skip to content

FIX: Converter Mixed Media Types #1719

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jbolor21 wants to merge 3 commits into
base: main
Choose a base branch
from
Open

FIX: Converter Mixed Media Types #1719

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d9264cd
add changes to files
jbolor21 May 11, 2026
3c7bb14
merge conflicts
jbolor21 May 11, 2026
6600ec4
add tests
jbolor21 May 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions frontend/src/components/Chat/ChatInputArea.styles.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,24 @@ export const useChatInputAreaStyles = makeStyles({
fontSize: tokens.fontSizeBase200,
color: tokens.colorNeutralForeground2,
},
openLink: {
display: 'inline-flex',
alignItems: 'center',
gap: tokens.spacingHorizontalXXS,
padding: `0 ${tokens.spacingHorizontalS}`,
borderRadius: tokens.borderRadiusSmall,
backgroundColor: tokens.colorBrandBackground,
color: tokens.colorNeutralForegroundOnBrand,
fontSize: tokens.fontSizeBase200,
fontWeight: tokens.fontWeightSemibold as unknown as string,
textDecoration: 'none',
flexShrink: 0,
height: '20px',
':hover': {
backgroundColor: tokens.colorBrandBackgroundHover,
color: tokens.colorNeutralForegroundOnBrand,
},
},
unsupportedWarning: {
display: 'flex',
alignItems: 'center',
Expand Down
50 changes: 48 additions & 2 deletions frontend/src/components/Chat/ChatInputArea.test.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -558,7 +558,7 @@ describe("ChatInputArea", () => {
<ChatInputArea
{...defaultProps}
activeTarget={{ target_registry_name: "t", target_type: "T", endpoint: "e", model_name: "m" }}
mediaConversions={[{ pieceType: "image", convertedValue: "/tmp/converted.png" }]}
mediaConversions={[{ pieceType: "image", convertedValue: "/tmp/converted.png", convertedDataType: "image_path" }]}
/>
</TestWrapper>
);
Expand Down Expand Up @@ -586,7 +586,7 @@ describe("ChatInputArea", () => {
<ChatInputArea
{...defaultProps}
activeTarget={{ target_registry_name: "t", target_type: "T", endpoint: "e", model_name: "m" }}
mediaConversions={[{ pieceType: "image", convertedValue: "/tmp/converted.png" }]}
mediaConversions={[{ pieceType: "image", convertedValue: "/tmp/converted.png", convertedDataType: "image_path" }]}
onClearMediaConversion={onClearMediaConversion}
/>
</TestWrapper>
Expand Down Expand Up @@ -663,6 +663,52 @@ describe("ChatInputArea", () => {
expect(onClearConversion).toHaveBeenCalled();
});

it("should render converted file chip with Open link for text→file conversion", async () => {
render(
<TestWrapper>
<ChatInputArea
{...defaultProps}
activeTarget={{ target_registry_name: "t", target_type: "T", endpoint: "e", model_name: "m" }}
convertedFileChip={{
name: "result.pdf",
url: "/api/media?path=%2Ftmp%2Fresult.pdf",
iconKind: "file",
}}
/>
</TestWrapper>
);

expect(screen.getByTestId("original-banner")).toBeInTheDocument();
const chip = screen.getByTestId("converted-file-chip");
expect(chip).toHaveTextContent("result.pdf");
const openLink = screen.getByTestId("converted-file-open");
expect(openLink).toHaveAttribute("href", "/api/media?path=%2Ftmp%2Fresult.pdf");
expect(openLink).toHaveAttribute("target", "_blank");
});

it("should call onClearConvertedFileChip when chip dismiss is clicked", async () => {
const onClearConvertedFileChip = jest.fn();
const user = userEvent.setup();

render(
<TestWrapper>
<ChatInputArea
{...defaultProps}
activeTarget={{ target_registry_name: "t", target_type: "T", endpoint: "e", model_name: "m" }}
convertedFileChip={{
name: "result.pdf",
url: "/api/media?path=%2Ftmp%2Fresult.pdf",
iconKind: "file",
}}
onClearConvertedFileChip={onClearConvertedFileChip}
/>
</TestWrapper>
);

await user.click(screen.getByTestId("clear-converted-file-chip"));
expect(onClearConvertedFileChip).toHaveBeenCalledTimes(1);
});

// ---------------------------------------------------------------------------
// Unsupported modality warnings
// ---------------------------------------------------------------------------
Expand Down
60 changes: 54 additions & 6 deletions frontend/src/components/Chat/ChatInputArea.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import {
tokens,
mergeClasses,
} from '@fluentui/react-components'
import { SendRegular, AttachRegular, DismissRegular, InfoRegular, AddRegular, CopyRegular, WarningRegular, SettingsRegular, ArrowShuffleRegular } from '@fluentui/react-icons'
import { SendRegular, AttachRegular, DismissRegular, InfoRegular, AddRegular, CopyRegular, WarningRegular, SettingsRegular, ArrowShuffleRegular, OpenRegular } from '@fluentui/react-icons'
import { MessageAttachment, TargetInstance } from '../../types'
import { useChatInputAreaStyles } from './ChatInputArea.styles'
import { PIECE_TYPE_TO_DATA_TYPE } from './converterTypes'
Expand All @@ -16,6 +16,12 @@ import { PIECE_TYPE_TO_DATA_TYPE } from './converterTypes'
// Reusable status banner
// ---------------------------------------------------------------------------

export interface ConvertedFileChip {
name: string
url: string
iconKind: 'image' | 'audio' | 'video' | 'file'
}

interface StatusBannerProps {
icon: React.ReactElement
text: string
Expand Down Expand Up @@ -55,7 +61,7 @@ function StatusBanner({ icon, text, buttonText, buttonIcon, onButtonClick, testI

interface AttachmentListProps {
attachments: MessageAttachment[]
mediaConversions: Array<{ pieceType: string; convertedValue: string }>
mediaConversions: Array<{ pieceType: string; convertedValue: string; convertedDataType: string }>
onRemove: (index: number) => void
onClearMediaConversion: (pieceType: string) => void
formatFileSize: (bytes: number) => string
Expand Down Expand Up @@ -120,21 +126,24 @@ function AttachmentList({ attachments, mediaConversions, onRemove, onClearMediaC
interface TextInputRowsProps {
input: string
convertedValue?: string | null
convertedFileChip?: ConvertedFileChip | null
disabled: boolean
textareaRef: Ref<HTMLTextAreaElement>
convertedRef: Ref<HTMLTextAreaElement>
onInput: (e: React.ChangeEvent<HTMLTextAreaElement>) => void
onKeyDown: (e: KeyboardEvent<HTMLTextAreaElement>) => void
onConvertedValueChange: (value: string) => void
onClearConvertedFileChip?: () => void
styles: ReturnType<typeof useChatInputAreaStyles>
textInputClassName: string
}

function TextInputRows({ input, convertedValue, disabled, textareaRef, convertedRef, onInput, onKeyDown, onConvertedValueChange, styles, textInputClassName }: TextInputRowsProps) {
function TextInputRows({ input, convertedValue, convertedFileChip, disabled, textareaRef, convertedRef, onInput, onKeyDown, onConvertedValueChange, onClearConvertedFileChip, styles, textInputClassName }: TextInputRowsProps) {
const hasConversion = Boolean(convertedValue) || Boolean(convertedFileChip)
return (
<>
<div className={styles.textRow}>
{convertedValue && (
{hasConversion && (
<span className={styles.originalBadge} data-testid="original-banner">Original</span>
)}
<textarea
Expand Down Expand Up @@ -162,6 +171,40 @@ function TextInputRows({ input, convertedValue, disabled, textareaRef, converted
/>
</div>
)}
{!convertedValue && convertedFileChip && (
<div className={styles.convertedRow} data-testid="converted-file-chip">
<span className={styles.convertedBadge}>Converted</span>
<span aria-hidden="true">
{convertedFileChip.iconKind === 'image' && '🖼️'}
{convertedFileChip.iconKind === 'audio' && '🎵'}
{convertedFileChip.iconKind === 'video' && '🎥'}
Comment on lines +178 to +180
Copy link
Copy Markdown
Contributor

@romanlutz romanlutz May 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we not render these? file is different, of course, but image, audio, video at least? Would be nice to be able to play audio/video and see image

{convertedFileChip.iconKind === 'file' && '📄'}
</span>
<Caption1 className={styles.convertedFilename} title={convertedFileChip.name}>
{convertedFileChip.name}
</Caption1>
<Tooltip content="Open in new tab" relationship="label">
<a
href={convertedFileChip.url}
target="_blank"
rel="noopener noreferrer"
className={styles.openLink}
data-testid="converted-file-open"
>
<OpenRegular fontSize={14} />
<span>Open</span>
</a>
</Tooltip>
<Button
appearance="transparent"
size="small"
className={styles.dismissBtn}
icon={<DismissRegular />}
onClick={onClearConvertedFileChip}
data-testid="clear-converted-file-chip"
/>
</div>
)}
</>
)
}
Expand Down Expand Up @@ -255,11 +298,14 @@ interface ChatInputAreaProps {
onClearConversion: () => void
onConvertedValueChange: (value: string) => void
converterOutputDataTypes?: string[]
mediaConversions?: Array<{ pieceType: string; convertedValue: string }>
mediaConversions?: Array<{ pieceType: string; convertedValue: string; convertedDataType: string }>
onClearMediaConversion: (pieceType: string) => void
/** Chip describing a text→file conversion (e.g. PDFConverter output). */
convertedFileChip?: ConvertedFileChip | null
onClearConvertedFileChip?: () => void
}

const ChatInputArea = forwardRef<ChatInputAreaHandle, ChatInputAreaProps>(function ChatInputArea({ onSend, disabled = false, activeTarget, singleTurnLimitReached = false, onNewConversation, operatorLocked = false, crossTargetLocked = false, onUseAsTemplate, attackOperator, noTargetSelected = false, onConfigureTarget, onToggleConverterPanel, isConverterPanelOpen = false, onInputChange, onAttachmentsChange, convertedValue, originalValue: _originalValue, onClearConversion, onConvertedValueChange, converterOutputDataTypes = [], mediaConversions = [], onClearMediaConversion }, ref) {
const ChatInputArea = forwardRef<ChatInputAreaHandle, ChatInputAreaProps>(function ChatInputArea({ onSend, disabled = false, activeTarget, singleTurnLimitReached = false, onNewConversation, operatorLocked = false, crossTargetLocked = false, onUseAsTemplate, attackOperator, noTargetSelected = false, onConfigureTarget, onToggleConverterPanel, isConverterPanelOpen = false, onInputChange, onAttachmentsChange, convertedValue, originalValue: _originalValue, onClearConversion, onConvertedValueChange, converterOutputDataTypes = [], mediaConversions = [], onClearMediaConversion, convertedFileChip, onClearConvertedFileChip }, ref) {
const styles = useChatInputAreaStyles()
const [input, setInput] = useState('')
const [attachments, setAttachments] = useState<MessageAttachment[]>([])
Expand Down Expand Up @@ -526,12 +572,14 @@ const ChatInputArea = forwardRef<ChatInputAreaHandle, ChatInputAreaProps>(functi
<TextInputRows
input={input}
convertedValue={convertedValue}
convertedFileChip={convertedFileChip}
disabled={disabled}
textareaRef={textareaRef}
convertedRef={convertedRef}
onInput={handleInput}
onKeyDown={handleKeyDown}
onConvertedValueChange={onConvertedValueChange}
onClearConvertedFileChip={onClearConvertedFileChip}
styles={styles}
textInputClassName={textInputClassName}
/>
Expand Down
154 changes: 154 additions & 0 deletions frontend/src/components/Chat/ChatWindow.test.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2559,4 +2559,158 @@ describe("ChatWindow Integration", () => {
expect(screen.getByTestId("converter-panel")).toBeInTheDocument();
});
});

// -----------------------------------------------------------------------
// Text → File converter flow (e.g. PDFConverter)
// -----------------------------------------------------------------------

it("should render converted-file chip and synthesize file attachment when a text→file converter is used", async () => {
mockedConvertersApi.listConverterCatalog.mockResolvedValue({
items: [
{
converter_type: "PDFConverter",
supported_input_types: ["text"],
supported_output_types: ["binary_path"],
parameters: [],
},
],
});
mockedConvertersApi.createConverter.mockResolvedValue({
converter_id: "conv-pdf",
converter_type: "PDFConverter",
});
mockedConvertersApi.previewConversion.mockResolvedValue({
converted_value: "/tmp/results/report.pdf",
converted_value_data_type: "binary_path",
});
mockedAttacksApi.createAttack.mockResolvedValue({
attack_result_id: "ar-pdf",
conversation_id: "conv-pdf-flow",
created_at: "2026-01-01T00:00:00Z",
} as never);
// Keep addMessage pending so the optimistic user message (with the
// synthesized file attachment) remains in the DOM for assertion.
mockedAttacksApi.addMessage.mockImplementation(
() => new Promise(() => {}) as never
);
mockedMapper.buildMessagePieces.mockResolvedValue([
{ piece_type: "text", original_value: "make a pdf" } as never,
]);
mockedMapper.backendMessagesToFrontend.mockReturnValue([]);

render(
<TestWrapper>
<ChatWindow {...defaultProps} conversationId={null} />
</TestWrapper>
);

// 1. Type input
const chatInput = screen.getByTestId("chat-input");
await userEvent.type(chatInput, "make a pdf");

// 2. Open converter panel and select PDFConverter
await userEvent.click(screen.getByTestId("toggle-converter-panel-btn"));
const combobox = screen.getByRole("combobox");
await userEvent.click(combobox);
const option = await screen.findByRole("option", { name: /PDFConverter/ });
await userEvent.click(option);

// 3. text→file converters do not auto-preview; click Preview explicitly
await waitFor(() => {
expect(screen.getByTestId("converter-preview-btn")).toBeInTheDocument();
});
await userEvent.click(screen.getByTestId("converter-preview-btn"));
await waitFor(() => {
expect(screen.getByTestId("converter-preview-result")).toBeInTheDocument();
});

// 4. Use the converted value — populates pieceConversions['text'] with binary_path output
await userEvent.click(screen.getByTestId("use-converted-btn"));

// 5. The file chip should appear in the input area (covers convertedFileChip IIFE)
const chip = await screen.findByTestId("converted-file-chip");
expect(chip).toHaveTextContent("report.pdf");
const openLink = screen.getByTestId("converted-file-open");
expect(openLink).toHaveAttribute(
"href",
expect.stringContaining(encodeURIComponent("/tmp/results/report.pdf"))
);

// 6. Send — covers handleSend's text→file branch (buildMediaUrl /
// dataTypeToAttachmentKind / basenameFromValue) which synthesizes a
// file attachment on the optimistic user message.
const sendBtn = screen.getByTestId("send-message-btn");
await waitFor(() => expect(sendBtn).toBeEnabled());
await userEvent.click(sendBtn);

await waitFor(() => {
expect(mockedAttacksApi.createAttack).toHaveBeenCalled();
});

// The optimistic user bubble carries the synthesized file attachment.
// MessageList renders the file attachment with a unique testid we can target.
const attachmentOpen = await screen.findByTestId("attachment-open-0-0");
expect(attachmentOpen).toHaveAttribute(
"href",
expect.stringContaining(encodeURIComponent("/tmp/results/report.pdf"))
);
});

it("should auto-clear a stale text→text conversion when the typed text diverges from the original", async () => {
mockedConvertersApi.listConverterCatalog.mockResolvedValue({
items: [
{
converter_type: "Base64Converter",
supported_input_types: ["text"],
supported_output_types: ["text"],
parameters: [],
},
],
});
mockedConvertersApi.createConverter.mockResolvedValue({
converter_id: "conv-b64-stale",
converter_type: "Base64Converter",
});
mockedConvertersApi.previewConversion.mockResolvedValue({
converted_value: "aGVsbG8=",
converted_value_data_type: "text",
});

render(
<TestWrapper>
<ChatWindow {...defaultProps} conversationId={null} />
</TestWrapper>
);

const chatInput = screen.getByTestId("chat-input");
await userEvent.type(chatInput, "hello");

await userEvent.click(screen.getByTestId("toggle-converter-panel-btn"));
const combobox = screen.getByRole("combobox");
await userEvent.click(combobox);
const option = await screen.findByRole("option", { name: /Base64Converter/ });
await userEvent.click(option);

await waitFor(() => {
expect(screen.getByTestId("converter-preview-btn")).toBeInTheDocument();
});
await userEvent.click(screen.getByTestId("converter-preview-btn"));
await waitFor(() => {
expect(screen.getByTestId("converter-preview-result")).toBeInTheDocument();
});
await userEvent.click(screen.getByTestId("use-converted-btn"));

// The converted text row now exists (originalValue captured = "hello")
await waitFor(() => {
expect(screen.getByTestId("converted-value-input")).toBeInTheDocument();
});

// Type more — originalValue no longer matches chatInputText, so the
// auto-clear effect must drop pieceConversions['text'].
await userEvent.type(chatInput, " world");

await waitFor(() => {
expect(screen.queryByTestId("converted-value-input")).not.toBeInTheDocument();
});
});
});
Loading
Loading