Skip to content

Commit 15c0b74

Browse files
committed
feat: improve AI image compression, add missing tools, and enhance MCP tooling
- Replace JPEG image compression with two-phase WebP strategy that preserves quality better (quality reduction first, then dimension scaling as last resort), bump limit to 200KB - Add Bash, TodoRead, TodoWrite, WebFetch, WebSearch to allowedTools to prevent Claude Code process exit code 1 from permission blocking - Add purePreview param to takeScreenshot MCP tool to capture live preview without element highlight overlays/toolboxes - Add logging to controlEditor for debugging file open failures
1 parent 9fd000b commit 15c0b74

6 files changed

Lines changed: 359 additions & 39 deletions

File tree

src-node/claude-code-agent.js

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ exports.checkAvailability = async function () {
132132
* aiProgress, aiTextStream, aiToolEdit, aiError, aiComplete
133133
*/
134134
exports.sendPrompt = async function (params) {
135-
const { prompt, projectPath, sessionAction, model, locale, selectionContext } = params;
135+
const { prompt, projectPath, sessionAction, model, locale, selectionContext, images } = params;
136136
const requestId = Date.now().toString(36) + Math.random().toString(36).slice(2, 7);
137137

138138
// Handle session
@@ -172,7 +172,7 @@ exports.sendPrompt = async function (params) {
172172
}
173173

174174
// Run the query asynchronously — don't await here so we return requestId immediately
175-
_runQuery(requestId, enrichedPrompt, projectPath, model, currentAbortController.signal, locale)
175+
_runQuery(requestId, enrichedPrompt, projectPath, model, currentAbortController.signal, locale, images)
176176
.catch(err => {
177177
console.error("[Phoenix AI] Query error:", err);
178178
});
@@ -220,7 +220,7 @@ exports.destroySession = async function () {
220220
/**
221221
* Internal: run a Claude SDK query and stream results back to the browser.
222222
*/
223-
async function _runQuery(requestId, prompt, projectPath, model, signal, locale) {
223+
async function _runQuery(requestId, prompt, projectPath, model, signal, locale, images) {
224224
let editCount = 0;
225225
let toolCounter = 0;
226226
let queryFn;
@@ -249,8 +249,10 @@ async function _runQuery(requestId, prompt, projectPath, model, signal, locale)
249249
cwd: projectPath || process.cwd(),
250250
maxTurns: undefined,
251251
allowedTools: [
252-
"Read", "Edit", "Write", "Glob", "Grep",
252+
"Read", "Edit", "Write", "Glob", "Grep", "Bash",
253253
"AskUserQuestion", "Task",
254+
"TodoRead", "TodoWrite",
255+
"WebFetch", "WebSearch",
254256
"mcp__phoenix-editor__getEditorState",
255257
"mcp__phoenix-editor__takeScreenshot",
256258
"mcp__phoenix-editor__execJsInLivePreview",
@@ -483,6 +485,7 @@ async function _runQuery(requestId, prompt, projectPath, model, signal, locale)
483485
queryOptions.model = model;
484486
}
485487

488+
486489
// Resume session if we have an existing one (already cleared if sessionAction was "new")
487490
if (currentSessionId) {
488491
queryOptions.resume = currentSessionId;
@@ -493,8 +496,28 @@ async function _runQuery(requestId, prompt, projectPath, model, signal, locale)
493496
try {
494497
_log("Query start:", JSON.stringify(prompt).slice(0, 80), "cwd=" + (projectPath || "?"));
495498

499+
// Build prompt: multi-modal with images, or plain string
500+
let sdkPrompt = prompt;
501+
if (images && images.length > 0) {
502+
const contentBlocks = [{ type: "text", text: prompt }];
503+
images.forEach(function (img) {
504+
contentBlocks.push({
505+
type: "image",
506+
source: { type: "base64", media_type: img.mediaType, data: img.base64Data }
507+
});
508+
});
509+
sdkPrompt = (async function* () {
510+
yield {
511+
type: "user",
512+
session_id: currentSessionId || "",
513+
message: { role: "user", content: contentBlocks },
514+
parent_tool_use_id: null
515+
};
516+
})();
517+
}
518+
496519
const result = queryFn({
497-
prompt: prompt,
520+
prompt: sdkPrompt,
498521
options: queryOptions
499522
});
500523

@@ -753,6 +776,9 @@ async function _runQuery(requestId, prompt, projectPath, model, signal, locale)
753776

754777
_log("Error:", errMsg.slice(0, 200));
755778

779+
// Clear session after error to prevent cascading failures from resuming a broken session
780+
currentSessionId = null;
781+
756782
nodeConnector.triggerPeer("aiError", {
757783
requestId: requestId,
758784
error: errMsg
@@ -761,7 +787,7 @@ async function _runQuery(requestId, prompt, projectPath, model, signal, locale)
761787
// Always send aiComplete after aiError so the UI exits streaming state
762788
nodeConnector.triggerPeer("aiComplete", {
763789
requestId: requestId,
764-
sessionId: currentSessionId
790+
sessionId: null
765791
});
766792
}
767793
}

src-node/mcp-editor-tools.js

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -68,12 +68,18 @@ function createEditorMcpServer(sdkModule, nodeConnector) {
6868
"Prefer capturing specific regions instead of the full page: " +
6969
"use selector '#panel-live-preview-frame' for the live preview content, " +
7070
"or '.editor-holder' for the code editor area. " +
71-
"Only omit the selector when you need to see the full application layout.",
72-
{ selector: z.string().optional().describe("CSS selector to capture a specific element. Use '#panel-live-preview-frame' for the live preview, '.editor-holder' for the code editor.") },
71+
"Only omit the selector when you need to see the full application layout. " +
72+
"Note: live preview screenshots may include Phoenix toolbox overlays on selected elements " +
73+
"and other editor UI elements. Use purePreview=true to temporarily hide these overlays.",
74+
{
75+
selector: z.string().optional().describe("CSS selector to capture a specific element. Use '#panel-live-preview-frame' for the live preview, '.editor-holder' for the code editor."),
76+
purePreview: z.boolean().optional().describe("When true, temporarily switches to preview mode to hide element highlight overlays and toolboxes before capturing, then restores the previous mode.")
77+
},
7378
async function (args) {
7479
try {
7580
const result = await nodeConnector.execPeer("takeScreenshot", {
76-
selector: args.selector || undefined
81+
selector: args.selector || undefined,
82+
purePreview: args.purePreview || false
7783
});
7884
if (result.base64) {
7985
return {
@@ -152,15 +158,20 @@ function createEditorMcpServer(sdkModule, nodeConnector) {
152158
const results = [];
153159
let hasError = false;
154160
for (const op of args.operations) {
161+
console.log("[Phoenix AI] controlEditor:", op.operation, op.filePath);
155162
try {
156163
const result = await nodeConnector.execPeer("controlEditor", op);
157164
results.push(result);
158165
if (!result.success) {
159166
hasError = true;
167+
console.warn("[Phoenix AI] controlEditor failed:", op.operation, op.filePath, result.error);
168+
} else {
169+
console.log("[Phoenix AI] controlEditor success:", op.operation, op.filePath);
160170
}
161171
} catch (err) {
162172
results.push({ success: false, error: err.message });
163173
hasError = true;
174+
console.error("[Phoenix AI] controlEditor error:", op.operation, op.filePath, err.message);
164175
}
165176
}
166177
return {

src/core-ai/AIChatPanel.js

Lines changed: 173 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,16 @@ define(function (require, exports, module) {
6565
let _livePreviewDismissed = false; // user dismissed live preview chip
6666
let $contextBar; // DOM ref
6767

68+
// Image paste state
69+
let _attachedImages = []; // [{dataUrl, mediaType, base64Data}]
70+
const MAX_IMAGES = 10;
71+
const MAX_IMAGE_BASE64_SIZE = 200 * 1024; // ~200KB base64
72+
const ALLOWED_IMAGE_TYPES = [
73+
"image/png", "image/jpeg", "image/gif", "image/webp", "image/svg+xml"
74+
];
75+
6876
// DOM references
69-
let $panel, $messages, $status, $statusText, $textarea, $sendBtn, $stopBtn;
77+
let $panel, $messages, $status, $statusText, $textarea, $sendBtn, $stopBtn, $imagePreview;
7078

7179
// Live DOM query for $messages — the cached $messages reference can become stale
7280
// after SidebarTabs reparents the panel. Use this for any deferred operations
@@ -89,6 +97,7 @@ define(function (require, exports, module) {
8997
'<span class="ai-status-text">' + Strings.AI_CHAT_THINKING + '</span>' +
9098
'</div>' +
9199
'<div class="ai-chat-input-area">' +
100+
'<div class="ai-chat-image-preview"></div>' +
92101
'<div class="ai-chat-context-bar"></div>' +
93102
'<div class="ai-chat-input-wrap">' +
94103
'<textarea class="ai-chat-textarea" placeholder="' + Strings.AI_CHAT_PLACEHOLDER + '" rows="1"></textarea>' +
@@ -182,6 +191,7 @@ define(function (require, exports, module) {
182191
$textarea = $panel.find(".ai-chat-textarea");
183192
$sendBtn = $panel.find(".ai-send-btn");
184193
$stopBtn = $panel.find(".ai-stop-btn");
194+
$imagePreview = $panel.find(".ai-chat-image-preview");
185195

186196
// Event handlers
187197
$sendBtn.on("click", _sendMessage);
@@ -211,6 +221,43 @@ define(function (require, exports, module) {
211221
this.style.height = Math.min(this.scrollHeight, 96) + "px"; // max ~6rem
212222
});
213223

224+
// Paste handler for images
225+
$textarea.on("paste", function (e) {
226+
const items = (e.originalEvent || e).clipboardData && (e.originalEvent || e).clipboardData.items;
227+
if (!items) {
228+
return;
229+
}
230+
let imageFound = false;
231+
for (let i = 0; i < items.length; i++) {
232+
const item = items[i];
233+
if (item.kind === "file" && ALLOWED_IMAGE_TYPES.indexOf(item.type) !== -1) {
234+
if (_attachedImages.length >= MAX_IMAGES) {
235+
break;
236+
}
237+
imageFound = true;
238+
const blob = item.getAsFile();
239+
const reader = new FileReader();
240+
reader.onload = function (ev) {
241+
const dataUrl = ev.target.result;
242+
const commaIdx = dataUrl.indexOf(",");
243+
const base64Data = dataUrl.slice(commaIdx + 1);
244+
if (base64Data.length > MAX_IMAGE_BASE64_SIZE) {
245+
// Resize oversized images via canvas
246+
_resizeImage(dataUrl, function (resized) {
247+
_addImageIfUnique(resized.dataUrl, resized.mediaType, resized.base64Data);
248+
});
249+
} else {
250+
_addImageIfUnique(dataUrl, item.type, base64Data);
251+
}
252+
};
253+
reader.readAsDataURL(blob);
254+
}
255+
}
256+
if (imageFound) {
257+
e.preventDefault();
258+
}
259+
});
260+
214261
// Track scroll position for auto-scroll
215262
$messages.on("scroll", function () {
216263
const el = $messages[0];
@@ -281,8 +328,7 @@ define(function (require, exports, module) {
281328
const $img = $('<img class="ai-tool-screenshot" src="data:image/png;base64,' + base64 + '">');
282329
$img.on("click", function (e) {
283330
e.stopPropagation();
284-
$img.toggleClass("expanded");
285-
_scrollToBottom();
331+
_showImageLightbox($img.attr("src"));
286332
});
287333
$img.on("load", function () {
288334
// Force scroll — the image load changes height after insertion,
@@ -462,6 +508,104 @@ define(function (require, exports, module) {
462508
$contextBar.toggleClass("has-chips", $contextBar.children().length > 0);
463509
}
464510

511+
/**
512+
* Add an image to _attachedImages if it's not a duplicate.
513+
*/
514+
function _addImageIfUnique(dataUrl, mediaType, base64Data) {
515+
const isDuplicate = _attachedImages.some(function (existing) {
516+
return existing.base64Data === base64Data;
517+
});
518+
if (!isDuplicate && _attachedImages.length < MAX_IMAGES) {
519+
_attachedImages.push({dataUrl: dataUrl, mediaType: mediaType, base64Data: base64Data});
520+
_renderImagePreview();
521+
}
522+
}
523+
524+
/**
525+
* Resize an image so its base64 data stays under MAX_IMAGE_BASE64_SIZE.
526+
* Two-phase strategy using WebP for better quality-per-byte:
527+
* Phase 1 — reduce quality at original dimensions.
528+
* Phase 2 — scale dimensions down (75%, then 50%) and retry quality steps.
529+
*/
530+
function _resizeImage(dataUrl, callback) {
531+
const img = new Image();
532+
img.onload = function () {
533+
const canvas = document.createElement("canvas");
534+
const ctx = canvas.getContext("2d");
535+
const qualitySteps = [0.92, 0.85, 0.75, 0.6, 0.45];
536+
const scaleSteps = [1, 0.75, 0.5];
537+
let result;
538+
539+
for (let s = 0; s < scaleSteps.length; s++) {
540+
const scale = scaleSteps[s];
541+
canvas.width = Math.round(img.width * scale);
542+
canvas.height = Math.round(img.height * scale);
543+
ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
544+
545+
for (let q = 0; q < qualitySteps.length; q++) {
546+
result = canvas.toDataURL("image/webp", qualitySteps[q]);
547+
if (result.split(",")[1].length <= MAX_IMAGE_BASE64_SIZE) {
548+
const base64 = result.split(",")[1];
549+
callback({dataUrl: result, mediaType: "image/webp", base64Data: base64});
550+
return;
551+
}
552+
}
553+
}
554+
555+
// Last resort: use the smallest result we got
556+
const base64 = result.split(",")[1];
557+
callback({dataUrl: result, mediaType: "image/webp", base64Data: base64});
558+
};
559+
img.src = dataUrl;
560+
}
561+
562+
/**
563+
* Show a lightbox overlay with the full-size image.
564+
*/
565+
function _showImageLightbox(src) {
566+
const $overlay = $(
567+
'<div class="ai-image-lightbox">' +
568+
'<img />' +
569+
'</div>'
570+
);
571+
$overlay.find("img").attr("src", src);
572+
$overlay.on("click", function () {
573+
$overlay.remove();
574+
});
575+
$panel.append($overlay);
576+
}
577+
578+
/**
579+
* Render the image preview strip from _attachedImages.
580+
*/
581+
function _renderImagePreview() {
582+
if (!$imagePreview) {
583+
return;
584+
}
585+
$imagePreview.empty();
586+
if (_attachedImages.length === 0) {
587+
$imagePreview.removeClass("has-images");
588+
return;
589+
}
590+
$imagePreview.addClass("has-images");
591+
_attachedImages.forEach(function (img, idx) {
592+
const $thumb = $(
593+
'<span class="ai-image-thumb">' +
594+
'<img />' +
595+
'<button class="ai-image-remove" title="' + Strings.AI_CHAT_IMAGE_REMOVE + '">&times;</button>' +
596+
'</span>'
597+
);
598+
$thumb.find("img").attr("src", img.dataUrl).on("click", function () {
599+
_showImageLightbox(img.dataUrl);
600+
});
601+
$thumb.find(".ai-image-remove").on("click", function () {
602+
_attachedImages.splice(idx, 1);
603+
_renderImagePreview();
604+
});
605+
$imagePreview.append($thumb);
606+
});
607+
}
608+
465609
/**
466610
* Send the current input as a message to Claude.
467611
*/
@@ -474,8 +618,16 @@ define(function (require, exports, module) {
474618
// Show "+ New" button once a conversation starts
475619
$panel.find(".ai-new-session-btn").show();
476620

621+
// Capture attached images before clearing
622+
const imagesForDisplay = _attachedImages.slice();
623+
const imagesPayload = _attachedImages.map(function (img) {
624+
return {mediaType: img.mediaType, base64Data: img.base64Data};
625+
});
626+
_attachedImages = [];
627+
_renderImagePreview();
628+
477629
// Append user message
478-
_appendUserMessage(text);
630+
_appendUserMessage(text, imagesForDisplay);
479631

480632
// Clear input
481633
$textarea.val("");
@@ -540,7 +692,8 @@ define(function (require, exports, module) {
540692
projectPath: projectPath,
541693
sessionAction: "continue",
542694
locale: brackets.getLocale(),
543-
selectionContext: selectionContext
695+
selectionContext: selectionContext,
696+
images: imagesPayload.length > 0 ? imagesPayload : undefined
544697
}).then(function (result) {
545698
_currentRequestId = result.requestId;
546699
console.log("[AI UI] RequestId:", result.requestId);
@@ -583,6 +736,8 @@ define(function (require, exports, module) {
583736
_cursorDismissed = false;
584737
_cursorDismissedLine = null;
585738
_livePreviewDismissed = false;
739+
_attachedImages = [];
740+
_renderImagePreview();
586741
SnapshotStore.reset();
587742
PhoenixConnectors.clearPreviousContentMap();
588743
if ($messages) {
@@ -1281,14 +1436,26 @@ define(function (require, exports, module) {
12811436

12821437
// --- DOM helpers ---
12831438

1284-
function _appendUserMessage(text) {
1439+
function _appendUserMessage(text, images) {
12851440
const $msg = $(
12861441
'<div class="ai-msg ai-msg-user">' +
12871442
'<div class="ai-msg-label">' + Strings.AI_CHAT_LABEL_YOU + '</div>' +
12881443
'<div class="ai-msg-content"></div>' +
12891444
'</div>'
12901445
);
12911446
$msg.find(".ai-msg-content").text(text);
1447+
if (images && images.length > 0) {
1448+
const $imgDiv = $('<div class="ai-user-images"></div>');
1449+
images.forEach(function (img) {
1450+
const $thumb = $('<img class="ai-user-image-thumb" />');
1451+
$thumb.attr("src", img.dataUrl);
1452+
$thumb.on("click", function () {
1453+
_showImageLightbox(img.dataUrl);
1454+
});
1455+
$imgDiv.append($thumb);
1456+
});
1457+
$msg.find(".ai-msg-content").append($imgDiv);
1458+
}
12921459
$messages.append($msg);
12931460
_scrollToBottom();
12941461
}

0 commit comments

Comments
 (0)