use std::{ fmt::{self, Display}, iter, mem, str::FromStr, sync::Arc, }; use base64::{Engine, prelude::BASE64_STANDARD}; use egui::{ Color32, ColorImage, CornerRadius, Event, Frame, Id, Mesh, PointerButton, Pos2, Rect, Sense, Shape, Stroke, TextureHandle, Theme, Ui, Vec2, emath::{self, TSTransform}, epaint::{Brush, RectShape, TessellationOptions, Tessellator, Vertex}, load::SizedTexture, }; use eyre::{Context, bail}; use eyre::{OptionExt, eyre}; use half::f16; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{ custom_code_block::try_from_custom_code_block, rasterizer::{self, rasterize, rasterize_onto}, }; use crate::{custom_code_block::write_custom_code_block, util::random_id}; const HANDWRITING_MIN_HEIGHT: f32 = 100.0; const HANDWRITING_BOTTOM_PADDING: f32 = 80.0; const HANDWRITING_MARGIN: f32 = 0.05; const HANDWRITING_LINE_SPACING: f32 = 36.0; pub const CODE_BLOCK_KEY: &str = "handwriting"; type StrokeBlendMode = rasterizer::blend::Normal; const TESSELATION_OPTIONS: TessellationOptions = TessellationOptions { feathering: true, feathering_size_in_pixels: 1.0, coarse_tessellation_culling: true, prerasterized_discs: true, round_text_to_pixels: true, round_line_segments_to_pixels: true, round_rects_to_pixels: true, debug_paint_text_rects: false, debug_paint_clip_rects: false, debug_ignore_clip_rects: false, bezier_tolerance: 0.1, epsilon: 1.0e-5, parallel_tessellation: true, validate_meshes: false, }; pub struct HandwritingStyle { pub stroke: Stroke, pub bg_line_stroke: Stroke, pub bg_color: Color32, pub animate: bool, } #[derive(serde::Deserialize, serde::Serialize)] #[serde(default)] pub struct Handwriting { #[serde(skip, default = "random_id")] id: Id, strokes: Vec>, /// The stroke that is currently being drawed. #[serde(skip)] current_stroke: Vec, /// The lines that have not been blitted to `texture` yet. #[serde(skip)] unblitted_lines: Vec<[Pos2; 2]>, height: f32, desired_height: f32, /// Tesselated mesh of all strokes #[serde(skip)] mesh: Arc, #[serde(skip)] texture: Option, #[serde(skip)] image: ColorImage, #[serde(skip)] refresh_texture: bool, /// Context of the last mesh render. #[serde(skip)] last_mesh_ctx: Option, } pub struct HandwritingResponse { pub changed: bool, } /// Context of a mesh render. #[derive(Clone, Copy, PartialEq)] struct MeshContext { /// Need to update the mesh when the stroke color changes. pub ui_theme: Theme, pub pixels_per_point: f32, /// Canvas size in points pub size: Vec2, pub stroke: Stroke, } /// Get [Painting::texture], initializing it if necessary. macro_rules! texture { ($self_:expr, $ui:expr, $mesh_context:expr) => {{ let ui: &Ui = $ui; let mesh_context: &MeshContext = $mesh_context; let image_size = mesh_context.pixel_size(); let new_image = || { let image = ColorImage::new(image_size, Color32::TRANSPARENT); ui.ctx() .load_texture("handwriting", image, Default::default()) }; let texture = $self_.texture.get_or_insert_with(new_image); if texture.size() != image_size { $self_.refresh_texture = true; // TODO: don't redraw the entire mesh, just blit the old texture onto the new one *texture = new_image() }; texture }}; } impl MeshContext { /// Calculate canvas size in pixels pub fn pixel_size(&self) -> [usize; 2] { let Vec2 { x, y } = self.size * self.pixels_per_point; [x, y].map(|f| f.ceil() as usize) } } impl Default for Handwriting { fn default() -> Self { Self { id: random_id(), strokes: Default::default(), current_stroke: Default::default(), height: HANDWRITING_MIN_HEIGHT, desired_height: HANDWRITING_MIN_HEIGHT, mesh: Default::default(), texture: None, image: ColorImage::new([0, 0], Color32::WHITE), refresh_texture: true, last_mesh_ctx: None, unblitted_lines: Default::default(), } } } impl Handwriting { pub fn ui_control( &mut self, style: Option<&mut HandwritingStyle>, ui: &mut egui::Ui, response: &mut HandwritingResponse, ) -> egui::Response { ui.horizontal(|ui| { if let Some(style) = style { ui.label("Stroke:"); ui.add(&mut style.stroke); ui.separator(); } if ui.button("Clear Painting").clicked() { self.strokes.clear(); self.refresh_texture = true; response.changed = true; } ui.add_enabled_ui(!self.strokes.is_empty(), |ui| { if ui.button("Undo").clicked() { self.strokes.pop(); self.refresh_texture = true; response.changed = true; } }); let vertex_count: usize = self.mesh.indices.len() / 3; ui.label(format!("vertices: {vertex_count}")); }) .response } fn commit_current_line(&mut self, response: &mut HandwritingResponse) { debug_assert!(!self.current_stroke.is_empty()); self.strokes.push(mem::take(&mut self.current_stroke)); response.changed = true; } pub fn ui_content( &mut self, style: &HandwritingStyle, ui: &mut Ui, hw_response: &mut HandwritingResponse, ) -> egui::Response { if style.animate { self.height = ui.ctx().animate_value_with_time( self.id.with("height animation"), self.desired_height, 0.4, ); } else { self.height = self.desired_height; } let desired_size = Vec2::new(ui.available_width(), self.height); let (response, painter) = ui.allocate_painter(desired_size, Sense::drag()); let mut response = response //.on_hover_cursor(CursorIcon::Crosshair) //.on_hover_and_drag_cursor(CursorIcon::None) ; let size = response.rect.size(); // Calculate matrices that convert between screen-space and image-space. // - image-space: 0,0 is the top-left of the texture. // - screen-space: 0,0 is the top-left of the window. // Both spaces use the same logical points, not pixels. let to_screen = emath::RectTransform::from_to(Rect::from_min_size(Pos2::ZERO, size), response.rect); let from_screen = to_screen.inverse(); // Was the user in the process of drawing a stroke last frame? let was_drawing = !self.current_stroke.is_empty(); // Is the user in the process of drawing a stroke now? let is_drawing = response.interact_pointer_pos().is_some(); if !is_drawing { if was_drawing { // commit current line self.commit_current_line(hw_response); response.mark_changed(); } // recalculate how tall the widget should be let lines_max_y = self .strokes .iter() .flatten() .map(|p| p.y + HANDWRITING_BOTTOM_PADDING) .fold(HANDWRITING_MIN_HEIGHT, |max, y| max.max(y)); // Change the height of the handwriting item. // We don't do this mid-stroke, only when the user e.g. lifts the pen. if self.desired_height != lines_max_y { self.desired_height = lines_max_y; response.mark_changed(); } } else { let events = ui.ctx().input(|input| { // If we are getting both MouseMoved and PointerMoved events, ignore the first. let mut events = input.raw.events.iter().peekable(); iter::from_fn(move || { let next = events.next()?; let Some(peek) = events.peek() else { return Some(next); }; match next { Event::PointerMoved(..) if matches!(peek, Event::MouseMoved(..)) => { let _ = events.next(); // drop the MouseMoved event Some(next) } Event::MouseMoved(..) if matches!(peek, Event::PointerMoved(..)) => { // return the peeked PointerMoved instead Some(events.next().expect("next is some")) } _ => Some(next), } }) .cloned() .filter(|event| { // FIXME: pinenote: PointerMoved are duplicated after the MouseMoved events cfg!(not(feature = "pinenote")) || !matches!(event, Event::PointerMoved(..)) }) .collect::>() }); // Process input events and turn them into strokes for event in events { let last_canvas_pos = self.current_stroke.last(); match event { Event::PointerMoved(new_position) => { let new_canvas_pos = from_screen * new_position; if let Some(&last_canvas_pos) = last_canvas_pos { if last_canvas_pos != new_canvas_pos { self.push_to_stroke(new_canvas_pos); response.mark_changed(); } } } Event::MouseMoved(mut delta) => { if delta.length() == 0.0 { continue; } // FIXME: pinenote: MouseMovement delta does *not* take into account screen // scaling and rotation, so unless you've scaling=1 and no rotation, the // MouseMoved values will be all wrong. if cfg!(feature = "pinenote") { delta /= 1.8; delta = -delta.rot90(); } if let Some(&last_canvas_pos) = last_canvas_pos { self.push_to_stroke(last_canvas_pos + delta); response.mark_changed(); } else { println!("Got `MouseMoved`, but have no previous pos"); } } Event::PointerButton { pos, button, pressed, modifiers: _, } => match (button, pressed) { (PointerButton::Primary, true) => { if last_canvas_pos.is_none() { self.current_stroke.push(from_screen * pos); } } (PointerButton::Primary, false) => { if last_canvas_pos.is_some() { self.push_to_stroke(from_screen * pos); self.commit_current_line(hw_response); response.mark_changed(); } // Stop reading events. // TODO: In theory, we can get multiple press->draw->release series // in the same frame. Should handle this. break; } (_, _) => continue, }, // Stop drawing after pointer disappears or the window is unfocused // TODO: In theory, we can get multiple press->draw->release series // in the same frame. Should handle this. Event::PointerGone | Event::WindowFocused(false) => { if !self.current_stroke.is_empty() { self.commit_current_line(hw_response); break; } } Event::WindowFocused(true) | Event::Copy | Event::Cut | Event::Paste(..) | Event::Text(..) | Event::Key { .. } | Event::Zoom(..) | Event::Ime(..) | Event::Touch { .. } | Event::MouseWheel { .. } | Event::Screenshot { .. } => continue, } } } // Draw the horizontal ruled lines (1..) .map(|n| n as f32 * HANDWRITING_LINE_SPACING) .take_while(|&y| y < size.y) .map(|y| { let l = to_screen * Pos2::new(HANDWRITING_MARGIN * size.x, y); let r = to_screen * Pos2::new((1.0 - HANDWRITING_MARGIN) * size.x, y); Shape::hline(l.x..=r.x, l.y, style.bg_line_stroke) }) .for_each(|shape| { painter.add(shape); }); // Get the dimensions of the image let mesh_rect = response .rect .with_max_y(response.rect.min.y + self.desired_height); // These are the values that, if changed, would require the mesh to be re-rendered. let new_context = MeshContext { ui_theme: ui.ctx().theme(), pixels_per_point: ui.pixels_per_point(), size: mesh_rect.size(), stroke: style.stroke, }; // Figure out if we need to re-rasterize the mesh. if Some(&new_context) != self.last_mesh_ctx.as_ref() { self.refresh_texture = true; } if self.refresh_texture { // ...if we do, rasterize the entire texture from scratch self.refresh_texture(style, new_context, ui); self.unblitted_lines.clear(); } else if !self.unblitted_lines.is_empty() { // ...if we don't, we can get away with only rasterizing the *new* lines onto the // existing texture. for [from, to] in std::mem::take(&mut self.unblitted_lines) { self.draw_line_to_texture(from, to, &new_context, ui); } self.unblitted_lines.clear(); } // Draw the texture if let Some(texture) = &self.texture { let texture = SizedTexture::new(texture.id(), texture.size_vec2()); let shape = RectShape { rect: mesh_rect, corner_radius: CornerRadius::ZERO, fill: Color32::WHITE, stroke: Stroke::NONE, stroke_kind: egui::StrokeKind::Inside, round_to_pixels: None, blur_width: 0.0, brush: Some(Arc::new(Brush { fill_texture_id: texture.id, uv: Rect { min: Pos2::ZERO, max: Pos2::new(1.0, 1.0), }, })), }; painter.add(shape); } response } fn refresh_texture( &mut self, style: &HandwritingStyle, mesh_context: MeshContext, ui: &mut Ui, ) { self.last_mesh_ctx = Some(mesh_context); self.refresh_texture = false; #[cfg(not(target_arch = "wasm32"))] let start_time = std::time::Instant::now(); let mut tesselator = Tessellator::new( mesh_context.pixels_per_point, TESSELATION_OPTIONS, Default::default(), // we don't tesselate fonts vec![], ); let mesh = Arc::make_mut(&mut self.mesh); mesh.clear(); self.strokes .iter() .chain([&self.current_stroke]) .filter(|stroke| stroke.len() >= 2) .map(|stroke| { //let points: Vec = stroke.iter().map(|&p| to_screen * p).collect(); egui::Shape::line(stroke.clone(), style.stroke) }) .for_each(|shape| { tesselator.tessellate_shape(shape, mesh); }); let texture = texture!(self, ui, &mesh_context); let triangles = mesh_triangles(&self.mesh); let [px_x, px_y] = mesh_context.pixel_size(); let point_to_pixel = TSTransform::from_scaling(mesh_context.pixels_per_point); self.image = rasterize::(px_x, px_y, point_to_pixel, triangles); texture.set(self.image.clone(), Default::default()); #[cfg(not(target_arch = "wasm32"))] { let elapsed = start_time.elapsed(); println!("refreshed mesh in {:.3}s", elapsed.as_secs_f32()); } } pub fn ui(&mut self, style: &HandwritingStyle, ui: &mut Ui) -> HandwritingResponse { let mut response = HandwritingResponse { changed: false }; ui.vertical_centered_justified(|ui| { self.ui_control(None, ui, &mut response); //ui.label("Paint with your mouse/touch!"); Frame::canvas(ui.style()) .corner_radius(20.0) .stroke(Stroke::new(5.0, Color32::from_black_alpha(40))) .fill(style.bg_color) .show(ui, |ui| { self.ui_content(style, ui, &mut response); }); }); response } fn push_to_stroke(&mut self, new_canvas_pos: Pos2) { if let Some(&last_canvas_pos) = self.current_stroke.last() { if last_canvas_pos == new_canvas_pos { return; } self.unblitted_lines.push([last_canvas_pos, new_canvas_pos]); } self.current_stroke.push(new_canvas_pos); } /// Draw a single line onto the existing texture. fn draw_line_to_texture( &mut self, from: Pos2, to: Pos2, mesh_context: &MeshContext, ui: &mut Ui, ) { let mut tesselator = Tessellator::new( mesh_context.pixels_per_point, TESSELATION_OPTIONS, Default::default(), // we don't tesselate fonts vec![], ); let mut mesh = Mesh::default(); let line = egui::Shape::line_segment([from, to], mesh_context.stroke); tesselator.tessellate_shape(line, &mut mesh); self.draw_mesh_to_texture(&mesh, mesh_context, ui); } /// Draw a single mesh onto the existing texture. fn draw_mesh_to_texture(&mut self, mesh: &Mesh, mesh_context: &MeshContext, ui: &mut Ui) { let triangles = mesh_triangles(mesh); let point_to_pixel = TSTransform::from_scaling(mesh_context.pixels_per_point); rasterize_onto::(&mut self.image, point_to_pixel, triangles); texture!(self, ui, mesh_context).set(self.image.clone(), Default::default()); } pub fn strokes(&self) -> &[Vec] { &self.strokes } #[cfg(test)] pub fn example() -> Self { Handwriting { strokes: vec![ vec![ Pos2::new(-1.0, 1.0), Pos2::new(3.0, 1.0), Pos2::new(3.0, 3.0), Pos2::new(1.5, 2.0), Pos2::new(0.0, 0.0), ], vec![ Pos2::new(3.0, 3.0), Pos2::new(-1.0, 1.0), Pos2::new(0.0, 0.0), Pos2::new(3.0, 1.0), ], ], ..Default::default() } } } impl Display for Handwriting { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut raw = vec![]; for stroke in &self.strokes { raw.push((stroke.len() as u16).to_le_bytes()); for position in stroke { let x = half::f16::from_f32(position.x); let y = half::f16::from_f32(position.y); raw.push(x.to_bits().to_le_bytes()); raw.push(y.to_bits().to_le_bytes()); } } let raw = raw.as_slice().as_bytes(); write_custom_code_block(f, CODE_BLOCK_KEY, BASE64_STANDARD.encode(raw)) } } impl FromStr for Handwriting { type Err = eyre::Report; fn from_str(s: &str) -> Result { let s = try_from_custom_code_block(CODE_BLOCK_KEY, s) .ok_or_eyre("Not a valid ```handwriting-block")?; let bytes = BASE64_STANDARD .decode(s) .wrap_err("Failed to decode painting data from base64")?; #[allow(non_camel_case_types)] type u16_le = [u8; 2]; #[allow(non_camel_case_types)] type f16_le = [u8; 2]; #[derive(FromBytes, KnownLayout, Immutable)] #[repr(C, packed)] struct Stroke { pub len: u16_le, pub positions: [f16_le], } let mut bytes = &bytes[..]; let mut strokes = vec![]; while !bytes.is_empty() { let header_len = size_of::(); if bytes.len() < header_len { bail!("Invalid remaining length: {}", bytes.len()); } let stroke = Stroke::ref_from_bytes(&bytes[..header_len]).expect("length is correct"); let len = usize::from(u16::from_le_bytes(stroke.len)); let len = len * size_of::() * 2; if bytes.len() < len { bail!("Invalid remaining length: {}", bytes.len()); } let (stroke, rest) = bytes.split_at(header_len + len); bytes = rest; let stroke = Stroke::ref_from_bytes(stroke) .map_err(|e| eyre!("Failed to decode stroke bytes: {e}"))?; let mut positions = stroke .positions .iter() .map(|&position| f16::from_bits(u16::from_le_bytes(position))); let mut stroke = vec![]; while let Some(x) = positions.next() { let Some(y) = positions.next() else { unreachable!("len is a multiple of two"); }; stroke.push(Pos2::new(x.into(), y.into())); } strokes.push(stroke); } Ok(Handwriting { strokes, ..Default::default() }) } } impl HandwritingStyle { pub fn from_theme(theme: Theme) -> Self { let stroke_color; let bg_color; let line_color; match theme { Theme::Dark => { stroke_color = Color32::WHITE; bg_color = Color32::from_gray(30); line_color = Color32::from_rgb(100, 100, 100); } Theme::Light => { stroke_color = Color32::BLACK; bg_color = Color32::WHITE; line_color = Color32::from_rgb(130, 130, 130); // TODO } } HandwritingStyle { stroke: Stroke::new(1.6, stroke_color), bg_color, bg_line_stroke: Stroke::new(0.5, line_color), animate: true, } } } fn mesh_triangles(mesh: &Mesh) -> impl Iterator { mesh.triangles() .map(|indices| indices.map(|i| &mesh.vertices[i as usize])) } #[cfg(test)] mod test { use std::str::FromStr; use super::Handwriting; #[test] fn serialize_handwriting() { let handwriting = Handwriting::example(); insta::assert_debug_snapshot!("handwriting example", handwriting.strokes); let serialized = handwriting.to_string(); insta::assert_snapshot!("serialized handwriting", serialized); let deserialized = Handwriting::from_str(&serialized).expect("Handwriting must de/serialize correctly"); insta::assert_debug_snapshot!("deserialized handwriting", deserialized.strokes); } }