Desktop: Execute editor and node graph natively

Author: TrueDoctor

Cargo.lock

@@ -9,17 +9,20 @@ edition = "2024"
 rust-version = "1.87"
 
 [features]
-# default = ["gpu"]
-# gpu = ["graphite-editor/gpu"]
+default = ["gpu"]
+gpu = ["graphite-editor/gpu"]
 
 [dependencies]
-# Local dependencies
-# graphite-editor = { path = "../editor", features = [
-# 	"gpu",
-# 	"ron",
-# 	"vello",
-# 	"decouple-execution",
-# ] }
+# # Local dependencies
+graphite-editor = { path = "../editor", features = [
+	"gpu",
+	"ron",
+	"vello",
+	"decouple-execution",
+] }
+graph-craft = { workspace = true }
+wgpu-executor = { workspace = true }
+
 wgpu = { workspace = true }
 winit = { workspace = true, features = ["serde"] }
 thiserror = { workspace = true }
@@ -29,4 +32,6 @@ include_dir = { workspace = true }
 tracing-subscriber = { workspace = true }
 tracing = { workspace = true }
 dirs = { workspace = true }
+ron = { workspace = true}
 bytemuck = { workspace = true }
+glam = { workspace = true }

desktop/Cargo.toml

@@ -2,6 +2,9 @@ use crate::CustomEvent;
 use crate::WindowSize;
 use crate::render::GraphicsState;
 use crate::render::WgpuContext;
+use graph_craft::wasm_application_io::WasmApplicationIo;
+use graphite_editor::application::Editor;
+use graphite_editor::messages::prelude::*;
 use std::sync::Arc;
 use std::sync::mpsc::Sender;
 use std::time::Duration;
@@ -21,11 +24,13 @@ pub(crate) struct WinitApp {
 	pub(crate) cef_context: cef::Context<cef::Initialized>,
 	pub(crate) window: Option<Arc<Window>>,
 	cef_schedule: Option<Instant>,
+	// Cached frame buffer from CEF, used to check if mouse is on a transparent pixel
 	_ui_frame_buffer: Option<wgpu::Texture>,
 	window_size_sender: Sender<WindowSize>,
 	_viewport_frame_buffer: Option<wgpu::Texture>,
 	graphics_state: Option<GraphicsState>,
 	wgpu_context: WgpuContext,
+	pub(crate) editor: Editor,
 }
 
 impl WinitApp {
@@ -39,8 +44,25 @@ impl WinitApp {
 			graphics_state: None,
 			window_size_sender,
 			wgpu_context,
+			editor: Editor::new(),
 		}
 	}
+
+	fn dispatch_message(&mut self, message: Message) {
+		let responses = self.editor.handle_message(message);
+		self.send_messages_to_editor(responses);
+	}
+
+	fn send_messages_to_editor(&mut self, responses: Vec<FrontendMessage>) {
+		if responses.is_empty() {
+			return;
+		}
+		let Ok(message) = ron::to_string(&responses) else {
+			tracing::error!("Failed to serialize Messages");
+			return;
+		};
+		self.cef_context.send_web_message(message.as_bytes());
+	}
 }
 
 impl ApplicationHandler<CustomEvent> for WinitApp {
@@ -49,16 +71,41 @@ impl ApplicationHandler<CustomEvent> for WinitApp {
 		let timeout = Instant::now() + Duration::from_millis(10);
 		let wait_until = timeout.min(self.cef_schedule.unwrap_or(timeout));
 		self.cef_context.work();
+
+		let (_has_run, texture) = futures::executor::block_on(graphite_editor::node_graph_executor::run_node_graph());
+		if _has_run {
+			let mut responses = VecDeque::new();
+			let err = self.editor.poll_node_graph_evaluation(&mut responses);
+			if let Err(e) = err {
+				tracing::error!("Error poling node graph: {}", e);
+			}
+			let frontend_messages = responses
+				.into_iter()
+				.flat_map(|response| if let Message::Frontend(frontend) = response { Some(frontend) } else { None })
+				.collect();
+			self.send_messages_to_editor(frontend_messages);
+		}
+		if let Some(texture) = texture
+			&& let Some(graphics_state) = &mut self.graphics_state
+		{
+			graphics_state.bind_viewport_texture(texture.texture.as_ref());
+		}
+
 		event_loop.set_control_flow(ControlFlow::WaitUntil(wait_until));
 	}
 
-	fn new_events(&mut self, _event_loop: &ActiveEventLoop, _cause: StartCause) {
+	fn new_events(&mut self, _event_loop: &ActiveEventLoop, cause: StartCause) {
 		if let Some(schedule) = self.cef_schedule
 			&& schedule < Instant::now()
 		{
 			self.cef_schedule = None;
 			self.cef_context.work();
 		}
+		if let StartCause::ResumeTimeReached { .. } = cause {
+			if let Some(window) = &self.window {
+				window.request_redraw();
+			}
+		}
 	}
 
 	fn resumed(&mut self, event_loop: &ActiveEventLoop) {
@@ -71,12 +118,67 @@ impl ApplicationHandler<CustomEvent> for WinitApp {
 				)
 				.unwrap(),
 		);
-		let graphics_state = GraphicsState::new(window.clone(), self.wgpu_context.clone());
+		let mut graphics_state = GraphicsState::new(window.clone(), self.wgpu_context.clone());
+
+		let mut test_data = vec![0u8; 800 * 600 * 4];
+
+		for y in 0..600 {
+			for x in 0..800 {
+				let idx = (y * 800 + x) * 4;
+				test_data[idx + 1] = (x * 255 / 800) as u8; // Blue
+				test_data[idx + 2] = (y * 255 / 600) as u8; // Green
+				test_data[idx] = 255; // Red
+				test_data[idx + 3] = 255; // Alpha
+			}
+		}
+
+		let texture = self.wgpu_context.device.create_texture(&wgpu::TextureDescriptor {
+			label: Some("Viewport Texture"),
+			size: wgpu::Extent3d {
+				width: 800,
+				height: 600,
+				depth_or_array_layers: 1,
+			},
+			mip_level_count: 1,
+			sample_count: 1,
+			dimension: wgpu::TextureDimension::D2,
+			format: wgpu::TextureFormat::Bgra8UnormSrgb,
+			usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
+			view_formats: &[],
+		});
+
+		self.wgpu_context.queue.write_texture(
+			wgpu::TexelCopyTextureInfo {
+				texture: &texture,
+				mip_level: 0,
+				origin: wgpu::Origin3d::ZERO,
+				aspect: wgpu::TextureAspect::All,
+			},
+			test_data.as_slice(),
+			wgpu::TexelCopyBufferLayout {
+				offset: 0,
+				bytes_per_row: Some(4 * 800),
+				rows_per_image: Some(600),
+			},
+			wgpu::Extent3d {
+				width: 800,
+				height: 600,
+				depth_or_array_layers: 1,
+			},
+		);
+
+		graphics_state.bind_viewport_texture(&texture);
 
 		self.window = Some(window);
 		self.graphics_state = Some(graphics_state);
 
 		tracing::info!("Winit window created and ready");
+
+		graphite_editor::application::set_uuid_seed(42);
+
+		let application_io = WasmApplicationIo::new_with_context(self.wgpu_context.clone());
+
+		futures::executor::block_on(graphite_editor::node_graph_executor::replace_application_io(application_io));
 	}
 
 	fn user_event(&mut self, _: &ActiveEventLoop, event: CustomEvent) {
@@ -97,6 +199,32 @@ impl ApplicationHandler<CustomEvent> for WinitApp {
 					self.cef_schedule = Some(instant);
 				}
 			}
+			CustomEvent::MessageReceived { message } => {
+				if let Message::InputPreprocessor(ipp_message) = &message {
+					if let Some(window) = &self.window {
+						window.request_redraw();
+					}
+					if let InputPreprocessorMessage::CurrentTime { .. } | InputPreprocessorMessage::PointerMove { .. } = &ipp_message {
+					} else {
+						// println!("got ipp message: {:?}", &ipp_message.to_discriminant());
+					}
+				}
+				if let Message::InputPreprocessor(InputPreprocessorMessage::BoundsOfViewports { bounds_of_viewports }) = &message {
+					if let Some(graphic_state) = &mut self.graphics_state {
+						let window_size = self.window.as_ref().unwrap().inner_size();
+						let window_size = glam::Vec2::new(window_size.width as f32, window_size.height as f32);
+						let top_left = bounds_of_viewports[0].top_left.as_vec2() / window_size;
+						let bottom_right = bounds_of_viewports[0].bottom_right.as_vec2() / window_size;
+						let offset = top_left.to_array();
+						let scale = (bottom_right - top_left).recip();
+						graphic_state.set_viewport_offset(offset);
+						graphic_state.set_viewport_scale(scale.to_array());
+					} else {
+						panic!("graphics state not intialized, viewport offset might be lost");
+					}
+				}
+				self.dispatch_message(message);
+			}
 		}
 	}
 

desktop/src/app.rs

@@ -120,5 +120,15 @@ impl CefEventHandler for CefHandler {
 		let _ = self.event_loop_proxy.send_event(CustomEvent::ScheduleBrowserWork(scheduled_time));
 	}
 
-	fn receive_web_message(&self, message: &[u8]) {}
+	fn receive_web_message(&self, message: &[u8]) {
+		let str = std::str::from_utf8(message).unwrap();
+		match ron::from_str(str) {
+			Ok(message) => {
+				let _ = self.event_loop_proxy.send_event(CustomEvent::MessageReceived { message });
+			}
+			Err(e) => {
+				tracing::error!("Failed to deserialize message {:?}", e)
+			}
+		}
+	}
 }

desktop/src/cef.rs

@@ -1,9 +1,6 @@
 use cef::rc::{ConvertReturnValue, Rc, RcImpl};
 use cef::sys::{_cef_render_process_handler_t, cef_base_ref_counted_t, cef_render_process_handler_t, cef_v8_propertyattribute_t, cef_v8_value_create_array_buffer_with_copy};
-use cef::{
-	CefString, ImplFrame, ImplRenderProcessHandler, ImplV8Context, ImplV8Value, V8Handler, V8Propertyattribute, V8Value, WrapRenderProcessHandler, v8_context_get_entered_context,
-	v8_value_create_function,
-};
+use cef::{CefString, ImplFrame, ImplRenderProcessHandler, ImplV8Context, ImplV8Value, V8Handler, V8Propertyattribute, V8Value, WrapRenderProcessHandler, v8_value_create_function};
 
 use crate::cef::ipc::{MessageType, UnpackMessage, UnpackedMessage};
 
@@ -61,7 +58,9 @@ impl ImplRenderProcessHandler for RenderProcessHandlerImpl {
 					cef_v8_propertyattribute_t::V8_PROPERTY_ATTRIBUTE_READONLY.wrap_result(),
 				);
 
-				frame.execute_java_script(Some(&CefString::from(function_call.as_str())), None, 0);
+				if global.value_bykey(Some(&CefString::from(function_name))).is_some() {
+					frame.execute_java_script(Some(&CefString::from(function_call.as_str())), None, 0);
+				}
 
 				if context.exit() == 0 {
 					tracing::error!("Failed to exit V8 context");

desktop/src/cef/internal/render_process_handler.rs

@@ -1,4 +1,4 @@
-use cef::{CefString, Frame, ImplBinaryValue, ImplBrowser, ImplFrame, ImplListValue, ImplProcessMessage, ImplV8Context, ProcessId, V8Context, rc::ConvertParam, sys::cef_process_id_t};
+use cef::{CefString, Frame, ImplBinaryValue, ImplBrowser, ImplFrame, ImplListValue, ImplProcessMessage, ImplV8Context, ProcessId, V8Context, sys::cef_process_id_t};
 
 use super::{Context, Initialized};
 

desktop/src/cef/ipc.rs

@@ -2,6 +2,7 @@ use std::fmt::Debug;
 use std::process::exit;
 use std::time::Instant;
 
+use graphite_editor::messages::prelude::Message;
 use tracing_subscriber::EnvFilter;
 use winit::event_loop::EventLoop;
 
@@ -20,6 +21,7 @@ mod dirs;
 pub(crate) enum CustomEvent {
 	UiUpdate(wgpu::Texture),
 	ScheduleBrowserWork(Instant),
+	MessageReceived { message: Message },
 }
 
 fn main() {
@@ -38,7 +40,7 @@ fn main() {
 
 	let (window_size_sender, window_size_receiver) = std::sync::mpsc::channel();
 
-	let wgpu_context = futures::executor::block_on(WgpuContext::new());
+	let wgpu_context = futures::executor::block_on(WgpuContext::new()).unwrap();
 	let cef_context = match cef_context.init(cef::CefHandler::new(window_size_receiver, event_loop.create_proxy(), wgpu_context.clone())) {
 		Ok(c) => c,
 		Err(cef::InitError::AlreadyRunning) => {
@@ -51,6 +53,8 @@ fn main() {
 		}
 	};
 
+	tracing::info!("Cef initialized successfully");
+
 	let mut winit_app = WinitApp::new(cef_context, window_size_sender, wgpu_context);
 
 	event_loop.run_app(&mut winit_app).unwrap();

desktop/src/main.rs

@@ -56,46 +56,7 @@ pub(crate) enum FrameBufferError {
 	InvalidSize { buffer_size: usize, expected_size: usize, width: usize, height: usize },
 }
 
-#[derive(Debug, Clone)]
-pub(crate) struct WgpuContext {
-	pub(crate) device: wgpu::Device,
-	pub(crate) queue: wgpu::Queue,
-	adapter: wgpu::Adapter,
-	instance: wgpu::Instance,
-}
-
-impl WgpuContext {
-	pub(crate) async fn new() -> Self {
-		let instance = wgpu::Instance::new(&wgpu::InstanceDescriptor {
-			backends: wgpu::Backends::PRIMARY,
-			..Default::default()
-		});
-
-		let adapter = instance
-			.request_adapter(&wgpu::RequestAdapterOptions {
-				power_preference: wgpu::PowerPreference::default(),
-				compatible_surface: None,
-				force_fallback_adapter: false,
-			})
-			.await
-			.unwrap();
-
-		let required_limits = adapter.limits();
-
-		let (device, queue) = adapter
-			.request_device(&wgpu::DeviceDescriptor {
-				label: None,
-				required_features: wgpu::Features::PUSH_CONSTANTS,
-				required_limits,
-				memory_hints: Default::default(),
-				trace: wgpu::Trace::Off,
-			})
-			.await
-			.unwrap();
-
-		Self { device, queue, adapter, instance }
-	}
-}
+pub use wgpu_executor::Context as WgpuContext;
 
 #[derive(Debug)]
 pub(crate) struct GraphicsState {

desktop/src/render.rs

@@ -883,6 +883,7 @@ impl MessageHandler<PortfolioMessage, PortfolioMessageContext<'_>> for Portfolio
 				responses.add(FrontendMessage::UpdateOpenDocumentsList { open_documents });
 			}
 			PortfolioMessage::UpdateVelloPreference => {
+				responses.add(FrontendMessage::UpdateViewportHolePunch { active: preferences.use_vello });
 				responses.add(NodeGraphMessage::RunDocumentGraph);
 				self.persistent_data.use_vello = preferences.use_vello;
 			}