From 4f21693772e4d59333e7cc08d70995033fb174de Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 25 Jul 2025 18:13:51 -0700 Subject: [PATCH 1/2] Create function for CEF to call --- frontend/wasm/src/native.rs | 135 ++++++++++++++++++++++++++++++++++-- package.json | 2 +- 2 files changed, 130 insertions(+), 7 deletions(-) diff --git a/frontend/wasm/src/native.rs b/frontend/wasm/src/native.rs index 53ed32a5e9..90d02bfed8 100644 --- a/frontend/wasm/src/native.rs +++ b/frontend/wasm/src/native.rs @@ -1,21 +1,74 @@ -use std::cell::RefCell; -use std::time::Duration; - use crate::Message; use editor::messages::prelude::*; -use wasm_bindgen::closure::Closure; +use graphene_std::Color; +use graphene_std::raster::Image; +use js_sys::{Object, Reflect}; +use serde::ser::Serialize; +use std::sync::atomic::{AtomicU64, Ordering}; use wasm_bindgen::prelude::wasm_bindgen; use wasm_bindgen::{JsCast, JsValue}; +use web_sys::{CanvasRenderingContext2d, HtmlCanvasElement, ImageData, window}; + +// TODO: Remove +static IMAGE_DATA_HASH: AtomicU64 = AtomicU64::new(0); +fn calculate_hash(t: &T) -> u64 { + use std::collections::hash_map::DefaultHasher; + use std::hash::Hasher; + let mut hasher = DefaultHasher::new(); + t.hash(&mut hasher); + hasher.finish() +} #[wasm_bindgen] #[derive(Clone)] -pub struct EditorHandle; +pub struct EditorHandle { + /// TODO: Remove + /// We current do frontend message in native -> serde serialize -> json string -> serde deserialize -> frontend message in wasm -> JSValue -> browser + /// We should do native -> V8Value -> browser. + frontend_message_handler_callback: js_sys::Function, +} #[wasm_bindgen] impl EditorHandle { #[wasm_bindgen(constructor)] pub fn new(frontend_message_handler_callback: js_sys::Function) -> Self { - EditorHandle + EditorHandle { frontend_message_handler_callback } + } + + /// TODO: Remove + #[wasm_bindgen(js_name = sendMessageToFrontendFromCEF)] + pub fn send_message_to_frontend_from_cef(&self, message: String) { + let Ok(mut message) = serde_json::from_str::(&message) else { return }; + + if let FrontendMessage::UpdateImageData { ref image_data } = message { + let new_hash = calculate_hash(image_data); + let prev_hash = IMAGE_DATA_HASH.load(Ordering::Relaxed); + + if new_hash != prev_hash { + render_image_data_to_canvases(image_data.as_slice()); + IMAGE_DATA_HASH.store(new_hash, Ordering::Relaxed); + } + return; + } + + if let FrontendMessage::UpdateDocumentLayerStructure { data_buffer } = message { + message = FrontendMessage::UpdateDocumentLayerStructureJs { data_buffer: data_buffer.into() }; + } + + let message_type = message.to_discriminant().local_name(); + + let serializer = serde_wasm_bindgen::Serializer::new().serialize_large_number_types_as_bigints(true); + let message_data = message.serialize(&serializer).expect("Failed to serialize FrontendMessage"); + + let js_return_value = self.frontend_message_handler_callback.call2(&JsValue::null(), &JsValue::from(message_type), &message_data); + + if let Err(error) = js_return_value { + error!( + "While handling FrontendMessage \"{:?}\", JavaScript threw an error: {:?}", + message.to_discriminant().local_name(), + error, + ) + } } } @@ -43,3 +96,73 @@ pub fn send_message_to_cef>(message: T) { // Call it with argument func.call1(&JsValue::NULL, &JsValue::from_str(&serialized_message)).expect("Function call failed"); } + +// TODO: Remove +fn render_image_data_to_canvases(image_data: &[(u64, Image)]) { + let window = match window() { + Some(window) => window, + None => { + error!("Cannot render canvas: window object not found"); + return; + } + }; + let document = window.document().expect("window should have a document"); + let window_obj = Object::from(window); + let image_canvases_key = JsValue::from_str("imageCanvases"); + + let canvases_obj = match Reflect::get(&window_obj, &image_canvases_key) { + Ok(obj) if !obj.is_undefined() && !obj.is_null() => obj, + _ => { + let new_obj = Object::new(); + if Reflect::set(&window_obj, &image_canvases_key, &new_obj).is_err() { + error!("Failed to create and set imageCanvases object on window"); + return; + } + new_obj.into() + } + }; + let canvases_obj = Object::from(canvases_obj); + + for (placeholder_id, image) in image_data.iter() { + let canvas_name = placeholder_id.to_string(); + let js_key = JsValue::from_str(&canvas_name); + + if Reflect::has(&canvases_obj, &js_key).unwrap_or(false) || image.width == 0 || image.height == 0 { + continue; + } + + let canvas: HtmlCanvasElement = document + .create_element("canvas") + .expect("Failed to create canvas element") + .dyn_into::() + .expect("Failed to cast element to HtmlCanvasElement"); + + canvas.set_width(image.width); + canvas.set_height(image.height); + + let context: CanvasRenderingContext2d = canvas + .get_context("2d") + .expect("Failed to get 2d context") + .expect("2d context was not found") + .dyn_into::() + .expect("Failed to cast context to CanvasRenderingContext2d"); + let u8_data: Vec = image.data.iter().flat_map(|color| color.to_rgba8_srgb()).collect(); + let clamped_u8_data = wasm_bindgen::Clamped(&u8_data[..]); + match ImageData::new_with_u8_clamped_array_and_sh(clamped_u8_data, image.width, image.height) { + Ok(image_data_obj) => { + if context.put_image_data(&image_data_obj, 0., 0.).is_err() { + error!("Failed to put image data on canvas for id: {placeholder_id}"); + } + } + Err(e) => { + error!("Failed to create ImageData for id: {placeholder_id}: {e:?}"); + } + } + + let js_value = JsValue::from(canvas); + + if Reflect::set(&canvases_obj, &js_key, &js_value).is_err() { + error!("Failed to set canvas '{canvas_name}' on imageCanvases object"); + } + } +} diff --git a/package.json b/package.json index 7197418fb0..9bdfdae2e2 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "scripts": { "---------- DEV SERVER ----------": "", "start": "cd frontend && npm start", - "start-desktop": "cd frontend && npm run build-dev && cargo run -p graphite-desktop", + "start-desktop": "cd frontend && npm run build-native && cargo run -p graphite-desktop", "profiling": "cd frontend && npm run profiling", "production": "cd frontend && npm run production", "---------- BUILDS ----------": "", From d2b3bc2a871d6b36b36897557bc91a9d471b2343 Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 25 Jul 2025 20:31:06 -0700 Subject: [PATCH 2/2] implement native -> browser communication --- desktop/src/app.rs | 20 +++- desktop/src/cef.rs | 4 +- desktop/src/cef/internal.rs | 1 + desktop/src/cef/internal/client.rs | 18 ++-- .../cef/internal/non_browser_v8_handler.rs | 7 +- desktop/src/cef/internal/utility.rs | 8 ++ frontend/src/App.svelte | 1 + frontend/wasm/src/native.rs | 96 ------------------- 8 files changed, 35 insertions(+), 120 deletions(-) create mode 100644 desktop/src/cef/internal/utility.rs diff --git a/desktop/src/app.rs b/desktop/src/app.rs index 9eebd77df0..cb0798d443 100644 --- a/desktop/src/app.rs +++ b/desktop/src/app.rs @@ -2,10 +2,11 @@ use crate::CustomEvent; use crate::WindowSize; use crate::render::GraphicsState; use crate::render::WgpuContext; +use ::cef::CefString; +use ::cef::ImplBrowser; +use ::cef::ImplFrame; use graphite_editor::application::Editor; -use graphite_editor::dispatcher::Dispatcher; use graphite_editor::messages::prelude::Message; -use std::collections::VecDeque; use std::sync::Arc; use std::sync::mpsc::Sender; use std::time::Duration; @@ -109,10 +110,21 @@ impl ApplicationHandler for WinitApp { tracing::error!("Message could not be deserialized: {:?}", message); return; }; - println!("Message received: {message:?}"); let responses = self.editor.handle_message(message); - println!("responses: {:?}", responses); // Send response to CEF + let Some(frame) = self.cef_context.browser.as_ref().unwrap().main_frame() else { + tracing::error!("Could not get frame after editor processed messages"); + return; + }; + for frontend_message in responses { + let Ok(serialized_message) = serde_json::to_string(&frontend_message) else { + tracing::error!("Failed to serialize frontend message in CustomEvent::MessageReceived"); + continue; + }; + let message = format!("window.handle.sendMessageToFrontendFromCEF(\'{serialized_message}\')"); + let code = CefString::from(message.as_str()); + frame.execute_java_script(Some(&code), None, 0); + } } } } diff --git a/desktop/src/cef.rs b/desktop/src/cef.rs index 64dc53d673..00a5cf0d07 100644 --- a/desktop/src/cef.rs +++ b/desktop/src/cef.rs @@ -20,7 +20,7 @@ pub(crate) trait CefEventHandler: Clone { /// [`_cef_browser_process_handler_t::on_schedule_message_pump_work`] for more documentation. fn schedule_cef_message_loop_work(&self, scheduled_time: Instant); - fn send_message_to_editior(&self, message: String); + fn send_message_to_editor(&self, message: String); } #[derive(Clone, Copy)] @@ -118,7 +118,7 @@ impl CefEventHandler for CefHandler { fn schedule_cef_message_loop_work(&self, scheduled_time: std::time::Instant) { let _ = self.event_loop_proxy.send_event(CustomEvent::ScheduleBrowserWork(scheduled_time)); } - fn send_message_to_editior(&self, message: String) { + fn send_message_to_editor(&self, message: String) { let _ = self.event_loop_proxy.send_event(CustomEvent::MessageReceived { message }); } } diff --git a/desktop/src/cef/internal.rs b/desktop/src/cef/internal.rs index 37e140e4bf..96f620cd5b 100644 --- a/desktop/src/cef/internal.rs +++ b/desktop/src/cef/internal.rs @@ -5,6 +5,7 @@ mod non_browser_app; mod non_browser_render_process_handler; mod non_browser_v8_handler; mod render_handler; +mod utility; pub(crate) use app::AppImpl; pub(crate) use client::ClientImpl; diff --git a/desktop/src/cef/internal/client.rs b/desktop/src/cef/internal/client.rs index 91f28442b7..cfc12aa9c8 100644 --- a/desktop/src/cef/internal/client.rs +++ b/desktop/src/cef/internal/client.rs @@ -30,25 +30,19 @@ impl ImplClient for ClientImpl { fn on_process_message_received( &self, - browser: Option<&mut cef::Browser>, - frame: Option<&mut cef::Frame>, - source_process: cef::ProcessId, + _browser: Option<&mut cef::Browser>, + _frame: Option<&mut cef::Frame>, + _source_process: cef::ProcessId, message: Option<&mut cef::ProcessMessage>, ) -> ::std::os::raw::c_int { let Some(message) = message else { - tracing::event!(tracing::Level::ERROR, "No message in RenderProcessHandlerImpl::on_process_message_received"); + tracing::error!("No message in RenderProcessHandlerImpl::on_process_message_received"); return 1; }; let pointer: *mut cef::sys::_cef_string_utf16_t = message.name().into(); - let message = unsafe { - let str = (*pointer).str_; - let len = (*pointer).length; - let slice = std::slice::from_raw_parts(str, len as usize); - String::from_utf16(slice).unwrap() - }; - - let _ = self.event_handler.send_message_to_editior(message); + let string_message = super::utility::pointer_to_string(pointer); + let _ = self.event_handler.send_message_to_editor(string_message); 0 } } diff --git a/desktop/src/cef/internal/non_browser_v8_handler.rs b/desktop/src/cef/internal/non_browser_v8_handler.rs index a06d437b1e..bca5c945e1 100644 --- a/desktop/src/cef/internal/non_browser_v8_handler.rs +++ b/desktop/src/cef/internal/non_browser_v8_handler.rs @@ -24,12 +24,7 @@ impl ImplV8Handler for NonBrowserV8HandlerImpl { let string = arguments.unwrap().first().unwrap().as_ref().unwrap().string_value(); let pointer: *mut cef::sys::_cef_string_utf16_t = string.into(); - let message = unsafe { - let str = (*pointer).str_; - let len = (*pointer).length; - let slice = std::slice::from_raw_parts(str, len); - String::from_utf16(slice).unwrap() - }; + let message = super::utility::pointer_to_string(pointer); let Some(mut process_message) = process_message_create(Some(&CefString::from(message.as_str()))) else { tracing::event!(tracing::Level::ERROR, "Failed to create process message"); diff --git a/desktop/src/cef/internal/utility.rs b/desktop/src/cef/internal/utility.rs new file mode 100644 index 0000000000..bb7273c38d --- /dev/null +++ b/desktop/src/cef/internal/utility.rs @@ -0,0 +1,8 @@ +pub fn pointer_to_string(pointer: *mut cef::sys::_cef_string_utf16_t) -> String { + unsafe { + let str = (*pointer).str_; + let len = (*pointer).length; + let slice = std::slice::from_raw_parts(str, len as usize); + String::from_utf16(slice).unwrap() + } +} diff --git a/frontend/src/App.svelte b/frontend/src/App.svelte index cb7a23e83c..5a3b9bb8b4 100644 --- a/frontend/src/App.svelte +++ b/frontend/src/App.svelte @@ -14,6 +14,7 @@ editor = createEditor(); + window.handle = editor.handle; // Auto save every 15 seconds autoSaveAllDocumentsId = setInterval(() => { editor?.handle.autoSaveAllDocuments(); diff --git a/frontend/wasm/src/native.rs b/frontend/wasm/src/native.rs index 90d02bfed8..8cc3b2a74a 100644 --- a/frontend/wasm/src/native.rs +++ b/frontend/wasm/src/native.rs @@ -1,23 +1,8 @@ use crate::Message; use editor::messages::prelude::*; -use graphene_std::Color; -use graphene_std::raster::Image; -use js_sys::{Object, Reflect}; use serde::ser::Serialize; -use std::sync::atomic::{AtomicU64, Ordering}; use wasm_bindgen::prelude::wasm_bindgen; use wasm_bindgen::{JsCast, JsValue}; -use web_sys::{CanvasRenderingContext2d, HtmlCanvasElement, ImageData, window}; - -// TODO: Remove -static IMAGE_DATA_HASH: AtomicU64 = AtomicU64::new(0); -fn calculate_hash(t: &T) -> u64 { - use std::collections::hash_map::DefaultHasher; - use std::hash::Hasher; - let mut hasher = DefaultHasher::new(); - t.hash(&mut hasher); - hasher.finish() -} #[wasm_bindgen] #[derive(Clone)] @@ -40,17 +25,6 @@ impl EditorHandle { pub fn send_message_to_frontend_from_cef(&self, message: String) { let Ok(mut message) = serde_json::from_str::(&message) else { return }; - if let FrontendMessage::UpdateImageData { ref image_data } = message { - let new_hash = calculate_hash(image_data); - let prev_hash = IMAGE_DATA_HASH.load(Ordering::Relaxed); - - if new_hash != prev_hash { - render_image_data_to_canvases(image_data.as_slice()); - IMAGE_DATA_HASH.store(new_hash, Ordering::Relaxed); - } - return; - } - if let FrontendMessage::UpdateDocumentLayerStructure { data_buffer } = message { message = FrontendMessage::UpdateDocumentLayerStructureJs { data_buffer: data_buffer.into() }; } @@ -96,73 +70,3 @@ pub fn send_message_to_cef>(message: T) { // Call it with argument func.call1(&JsValue::NULL, &JsValue::from_str(&serialized_message)).expect("Function call failed"); } - -// TODO: Remove -fn render_image_data_to_canvases(image_data: &[(u64, Image)]) { - let window = match window() { - Some(window) => window, - None => { - error!("Cannot render canvas: window object not found"); - return; - } - }; - let document = window.document().expect("window should have a document"); - let window_obj = Object::from(window); - let image_canvases_key = JsValue::from_str("imageCanvases"); - - let canvases_obj = match Reflect::get(&window_obj, &image_canvases_key) { - Ok(obj) if !obj.is_undefined() && !obj.is_null() => obj, - _ => { - let new_obj = Object::new(); - if Reflect::set(&window_obj, &image_canvases_key, &new_obj).is_err() { - error!("Failed to create and set imageCanvases object on window"); - return; - } - new_obj.into() - } - }; - let canvases_obj = Object::from(canvases_obj); - - for (placeholder_id, image) in image_data.iter() { - let canvas_name = placeholder_id.to_string(); - let js_key = JsValue::from_str(&canvas_name); - - if Reflect::has(&canvases_obj, &js_key).unwrap_or(false) || image.width == 0 || image.height == 0 { - continue; - } - - let canvas: HtmlCanvasElement = document - .create_element("canvas") - .expect("Failed to create canvas element") - .dyn_into::() - .expect("Failed to cast element to HtmlCanvasElement"); - - canvas.set_width(image.width); - canvas.set_height(image.height); - - let context: CanvasRenderingContext2d = canvas - .get_context("2d") - .expect("Failed to get 2d context") - .expect("2d context was not found") - .dyn_into::() - .expect("Failed to cast context to CanvasRenderingContext2d"); - let u8_data: Vec = image.data.iter().flat_map(|color| color.to_rgba8_srgb()).collect(); - let clamped_u8_data = wasm_bindgen::Clamped(&u8_data[..]); - match ImageData::new_with_u8_clamped_array_and_sh(clamped_u8_data, image.width, image.height) { - Ok(image_data_obj) => { - if context.put_image_data(&image_data_obj, 0., 0.).is_err() { - error!("Failed to put image data on canvas for id: {placeholder_id}"); - } - } - Err(e) => { - error!("Failed to create ImageData for id: {placeholder_id}: {e:?}"); - } - } - - let js_value = JsValue::from(canvas); - - if Reflect::set(&canvases_obj, &js_key, &js_value).is_err() { - error!("Failed to set canvas '{canvas_name}' on imageCanvases object"); - } - } -}