mod canvas_rasterizer; mod disk_format; use std::{ fmt::{self, Display}, iter, mem, str::FromStr, sync::Arc, }; use arboard::Clipboard; use base64::{Engine, prelude::BASE64_STANDARD}; use canvas_rasterizer::CanvasRasterizer; use disk_format::{DiskFormat, RawStroke, RawStrokeHeader, f16_le}; use egui::{ Color32, Event, Frame, Id, Mesh, PointerButton, Pos2, Rect, Sense, Shape, Stroke, Theme, Ui, Vec2, emath::{self, TSTransform}, epaint::{TessellationOptions, Tessellator, Vertex}, }; use eyre::{Context, bail}; use eyre::{OptionExt, eyre}; use half::f16; use zerocopy::{FromBytes, IntoBytes}; use crate::{custom_code_block::try_from_custom_code_block, rasterizer}; use crate::{custom_code_block::write_custom_code_block, util::random_id}; const HANDWRITING_MIN_HEIGHT: f32 = 100.0; const HANDWRITING_BOTTOM_PADDING: f32 = 80.0; const HANDWRITING_MARGIN: f32 = 0.05; const HANDWRITING_LINE_SPACING: f32 = 36.0; pub const CODE_BLOCK_KEY: &str = "handwriting"; type StrokeBlendMode = rasterizer::blend::Normal; const TESSELLATION_OPTIONS: TessellationOptions = TessellationOptions { feathering: true, feathering_size_in_pixels: 1.0, coarse_tessellation_culling: true, prerasterized_discs: true, round_text_to_pixels: true, round_line_segments_to_pixels: true, round_rects_to_pixels: true, debug_paint_text_rects: false, debug_paint_clip_rects: false, debug_ignore_clip_rects: false, bezier_tolerance: 0.1, epsilon: 1.0e-5, parallel_tessellation: true, validate_meshes: false, }; pub struct HandwritingStyle { pub stroke: Stroke, pub bg_line_stroke: Stroke, pub bg_color: Color32, pub animate: bool, pub hide_cursor: bool, } #[derive(serde::Deserialize, serde::Serialize)] #[serde(default)] pub struct Handwriting { strokes: Vec>, height: f32, desired_height: f32, #[serde(skip)] e: Ephemeral, } /// Handwriting data that isn't persisted across restarts. struct Ephemeral { id: Id, canvas_rasterizer: CanvasRasterizer, /// The stroke that is currently being drawed. current_stroke: Vec, /// The lines that have not been blitted to `texture` yet. unblitted_lines: Vec<[Pos2; 2]>, tessellator: Option, /// Tessellated mesh of all strokes mesh: Arc, refresh_texture: bool, /// Context of the last mesh render. last_mesh_ctx: Option, } pub struct HandwritingResponse { pub changed: bool, } /// Context of a mesh render. #[derive(Clone, Copy, PartialEq)] struct MeshContext { /// Need to update the mesh when the stroke color changes. pub ui_theme: Theme, pub pixels_per_point: f32, pub stroke: Stroke, } impl Default for Handwriting { fn default() -> Self { Self { strokes: Default::default(), height: HANDWRITING_MIN_HEIGHT, desired_height: HANDWRITING_MIN_HEIGHT, e: Default::default(), } } } impl Default for Ephemeral { fn default() -> Self { Self { id: random_id(), canvas_rasterizer: Default::default(), current_stroke: Default::default(), tessellator: None, mesh: Default::default(), refresh_texture: true, last_mesh_ctx: None, unblitted_lines: Default::default(), } } } impl Handwriting { pub fn ui_control( &mut self, style: Option<&mut HandwritingStyle>, ui: &mut egui::Ui, response: &mut HandwritingResponse, ) -> egui::Response { ui.horizontal(|ui| { if let Some(style) = style { ui.label("Stroke:"); ui.add(&mut style.stroke); ui.separator(); } if ui.button("clear").clicked() { self.strokes.clear(); self.e.refresh_texture = true; response.changed = true; } ui.add_enabled_ui(!self.strokes.is_empty(), |ui| { if ui.button("undo").clicked() { self.strokes.pop(); self.e.refresh_texture = true; response.changed = true; } }); if ui.button("copy").clicked() { let text = self.to_string(); // TODO: move to a job let _ = Clipboard::new().unwrap().set_text(text); } let vertex_count: usize = self.e.mesh.indices.len() / 3; ui.label(format!("vertices: {vertex_count}")); }) .response } pub fn ui_content( &mut self, style: &HandwritingStyle, ui: &mut Ui, hw_response: &mut HandwritingResponse, ) -> egui::Response { if style.animate { self.height = ui.ctx().animate_value_with_time( self.e.id.with("height animation"), self.desired_height, 0.4, ); } else { self.height = self.desired_height; } let desired_size = Vec2::new(ui.available_width(), self.height); let (mut response, painter) = ui.allocate_painter(desired_size, Sense::drag()); if style.hide_cursor { response = response.on_hover_and_drag_cursor(egui::CursorIcon::None); } let size = response.rect.size(); // Calculate matrices that convert between screen-space and image-space. // - image-space: 0,0 is the top-left of the texture. // - screen-space: 0,0 is the top-left of the window. // Both spaces use the same logical points, not pixels. let to_screen = emath::RectTransform::from_to(Rect::from_min_size(Pos2::ZERO, size), response.rect); let from_screen = to_screen.inverse(); // Was the user in the process of drawing a stroke last frame? let was_drawing = !self.e.current_stroke.is_empty(); // Is the user in the process of drawing a stroke now? let is_drawing = response.interact_pointer_pos().is_some(); if !is_drawing { if was_drawing { // commit current line self.commit_current_line(hw_response); response.mark_changed(); } // recalculate how tall the widget should be let lines_max_y = self .strokes .iter() .flatten() .map(|p| p.y + HANDWRITING_BOTTOM_PADDING) .fold(HANDWRITING_MIN_HEIGHT, |max, y| max.max(y)); // Change the height of the handwriting item. // We don't do this mid-stroke, only when the user e.g. lifts the pen. if self.desired_height != lines_max_y { self.desired_height = lines_max_y; response.mark_changed(); } } else { let events = ui.ctx().input(|input| { // If we are getting both MouseMoved and PointerMoved events, ignore the first. let mut events = input.raw.events.iter().peekable(); iter::from_fn(move || { let next = events.next()?; let Some(peek) = events.peek() else { return Some(next); }; match next { Event::PointerMoved(..) if matches!(peek, Event::MouseMoved(..)) => { let _ = events.next(); // drop the MouseMoved event Some(next) } Event::MouseMoved(..) if matches!(peek, Event::PointerMoved(..)) => { // return the peeked PointerMoved instead Some(events.next().expect("next is some")) } _ => Some(next), } }) .filter(|event| { // FIXME: pinenote: PointerMoved are duplicated after the MouseMoved events cfg!(not(feature = "pinenote")) || !matches!(event, Event::PointerMoved(..)) }) .cloned() .collect::>() }); // Process input events and turn them into strokes for event in events { let last_canvas_pos = self.e.current_stroke.last(); match event { Event::PointerMoved(new_position) => { let new_canvas_pos = from_screen * new_position; if let Some(&last_canvas_pos) = last_canvas_pos { if last_canvas_pos != new_canvas_pos { self.push_to_stroke(new_canvas_pos); response.mark_changed(); } } } Event::MouseMoved(mut delta) => { if delta.length() == 0.0 { continue; } // FIXME: pinenote: MouseMovement delta does *not* take into account screen // scaling and rotation, so unless you've scaling=1 and no rotation, the // MouseMoved values will be all wrong. if cfg!(feature = "pinenote") { delta /= 1.8; delta = -delta.rot90(); } if let Some(&last_canvas_pos) = last_canvas_pos { self.push_to_stroke(last_canvas_pos + delta); response.mark_changed(); } else { println!("Got `MouseMoved`, but have no previous pos"); } } Event::PointerButton { pos, button, pressed, modifiers: _, } => match (button, pressed) { (PointerButton::Primary, true) => { if last_canvas_pos.is_none() { self.e.current_stroke.push(from_screen * pos); } } (PointerButton::Primary, false) => { if last_canvas_pos.is_some() { self.push_to_stroke(from_screen * pos); self.commit_current_line(hw_response); response.mark_changed(); } // Stop reading events. // TODO: In theory, we can get multiple press->draw->release series // in the same frame. Should handle this. break; } (_, _) => continue, }, // Stop drawing after pointer disappears or the window is unfocused // TODO: In theory, we can get multiple press->draw->release series // in the same frame. Should handle this. Event::PointerGone | Event::WindowFocused(false) => { if !self.e.current_stroke.is_empty() { self.commit_current_line(hw_response); break; } } Event::WindowFocused(true) | Event::Copy | Event::Cut | Event::Paste(..) | Event::Text(..) | Event::Key { .. } | Event::Zoom(..) | Event::Ime(..) | Event::Touch { .. } | Event::MouseWheel { .. } | Event::Screenshot { .. } => continue, } } } // Draw the horizontal ruled lines (1..) .map(|n| n as f32 * HANDWRITING_LINE_SPACING) .take_while(|&y| y < size.y) .map(|y| { let l = to_screen * Pos2::new(HANDWRITING_MARGIN * size.x, y); let r = to_screen * Pos2::new((1.0 - HANDWRITING_MARGIN) * size.x, y); Shape::hline(l.x..=r.x, l.y, style.bg_line_stroke) }) .for_each(|shape| { painter.add(shape); }); // Get the position and dimensions of the image let mesh_rect = response .rect .with_max_y(response.rect.min.y + self.desired_height); // These are the values that, if changed, would require the mesh to be re-rendered. let new_context = MeshContext { ui_theme: ui.ctx().theme(), pixels_per_point: ui.pixels_per_point(), stroke: style.stroke, }; // Figure out if we need to re-rasterize the mesh. if Some(&new_context) != self.e.last_mesh_ctx.as_ref() { self.e.refresh_texture = true; } let [px_width, px_height] = { let Vec2 { x, y } = mesh_rect.size() * new_context.pixels_per_point; [x, y].map(|f| f.ceil() as usize) }; self.e.canvas_rasterizer.set_size(px_width, px_height); if self.e.refresh_texture { // ...if we do, rasterize the entire texture from scratch self.refresh_texture(style, new_context); self.e.unblitted_lines.clear(); } else if !self.e.unblitted_lines.is_empty() { // ...if we don't, we can get away with only rasterizing the *new* lines onto the // existing texture. for [from, to] in std::mem::take(&mut self.e.unblitted_lines) { self.draw_line_to_texture(from, to, &new_context); } self.e.unblitted_lines.clear(); } // Draw the texture self.e.canvas_rasterizer.show(ui.ctx(), &painter, mesh_rect); response } fn commit_current_line(&mut self, response: &mut HandwritingResponse) { debug_assert!(!self.e.current_stroke.is_empty()); self.strokes.push(mem::take(&mut self.e.current_stroke)); response.changed = true; } /// Tessellate and rasterize the strokes into a new texture. fn refresh_texture(&mut self, style: &HandwritingStyle, mesh_context: MeshContext) { let Ephemeral { current_stroke, tessellator, mesh, refresh_texture, last_mesh_ctx, .. } = &mut self.e; // TODO: don't tessellate and rasterize on the GUI thread *last_mesh_ctx = Some(mesh_context); *refresh_texture = false; #[cfg(not(target_arch = "wasm32"))] let start_time = std::time::Instant::now(); let mesh = Arc::make_mut(mesh); mesh.clear(); // TODO: re-use tessellator if pixels_per_point hasn't changed let tessellator = tessellator.insert(new_tessellator(mesh_context.pixels_per_point)); self.strokes .iter() .chain([&*current_stroke]) .filter(|stroke| stroke.len() >= 2) .map(|stroke| { //let points: Vec = stroke.iter().map(|&p| to_screen * p).collect(); egui::Shape::line(stroke.clone(), style.stroke) }) .for_each(|shape| { tessellator.tessellate_shape(shape, mesh); }); // sanity-check that tessellation did not produce any NaNs. // this can happen if the line contains duplicated consecutive positions //for vertex in &mesh.vertices { // debug_assert!(vertex.pos.x.is_finite(), "{} must be finite", vertex.pos.x); // debug_assert!(vertex.pos.y.is_finite(), "{} must be finite", vertex.pos.y); //} let point_to_pixel = TSTransform::from_scaling(mesh_context.pixels_per_point); let triangles = mesh_triangles(&self.e.mesh); self.e.canvas_rasterizer.clear(); self.e .canvas_rasterizer .rasterize(point_to_pixel, triangles); #[cfg(not(target_arch = "wasm32"))] { let elapsed = start_time.elapsed(); log::debug!("refreshed mesh in {:.3}s", elapsed.as_secs_f32()); } } pub fn ui(&mut self, style: &HandwritingStyle, ui: &mut Ui) -> HandwritingResponse { let mut response = HandwritingResponse { changed: false }; ui.vertical_centered_justified(|ui| { self.ui_control(None, ui, &mut response); //ui.label("Paint with your mouse/touch!"); Frame::canvas(ui.style()) .corner_radius(20.0) .stroke(Stroke::new(5.0, Color32::from_black_alpha(40))) .fill(style.bg_color) .show(ui, |ui| { self.ui_content(style, ui, &mut response); }); }); response } /// Append a new [Pos2] to [Self::current_stroke]. /// /// Queue a new line to be drawn onto [Self::texture]. fn push_to_stroke(&mut self, new_canvas_pos: Pos2) { if let Some(&last_canvas_pos) = self.e.current_stroke.last() { if last_canvas_pos == new_canvas_pos { return; } self.e .unblitted_lines .push([last_canvas_pos, new_canvas_pos]); } self.e.current_stroke.push(new_canvas_pos); } /// Draw a single line onto the existing texture. fn draw_line_to_texture(&mut self, from: Pos2, to: Pos2, mesh_context: &MeshContext) { // INVARIANT: if this function was called, then pixels_per_point is the same as last frame, // so there's no need to create a new tessellator. let tessellator = self .e .tessellator .get_or_insert_with(|| new_tessellator(mesh_context.pixels_per_point)); let mut mesh = Mesh::default(); let line = egui::Shape::line_segment([from, to], mesh_context.stroke); tessellator.tessellate_shape(line, &mut mesh); self.draw_mesh_to_texture(&mesh, mesh_context); } /// Draw a single mesh onto the existing texture. fn draw_mesh_to_texture(&mut self, mesh: &Mesh, mesh_context: &MeshContext) { let triangles = mesh_triangles(mesh); let point_to_pixel = TSTransform::from_scaling(mesh_context.pixels_per_point); self.e .canvas_rasterizer .rasterize(point_to_pixel, triangles); } pub fn strokes(&self) -> &[Vec] { &self.strokes } #[cfg(test)] pub fn example() -> Self { Handwriting { strokes: vec![ vec![ Pos2::new(-1.0, 1.0), Pos2::new(3.0, 1.0), Pos2::new(3.0, 3.0), Pos2::new(1.5, 2.0), Pos2::new(0.0, 0.0), ], vec![ Pos2::new(3.0, 3.0), Pos2::new(-1.0, 1.0), Pos2::new(0.0, 0.0), Pos2::new(3.0, 1.0), ], ], ..Default::default() } } pub fn encode_as_disk_format(&self) -> Box<[u8]> { let mut bytes = vec![]; let header = disk_format::Header { version: disk_format::V1, }; bytes.extend_from_slice(header.as_bytes()); for stroke in &self.strokes { let Ok(len) = u16::try_from(stroke.len()) else { log::error!("More than u16::MAX points in a stroke!"); continue; }; let header = RawStrokeHeader { len: len.into() }; bytes.extend_from_slice(header.as_bytes()); for position in stroke { for v in [position.x, position.y] { let v = f16::from_f32(v); let v = f16_le::from(v); bytes.extend_from_slice(v.as_bytes()); } } } bytes.into_boxed_slice() } } fn new_tessellator(pixels_per_point: f32) -> Tessellator { Tessellator::new( pixels_per_point, TESSELLATION_OPTIONS, Default::default(), // we don't tessellate fonts vec![], ) } impl Display for Handwriting { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let raw = self.encode_as_disk_format(); write_custom_code_block(f, CODE_BLOCK_KEY, BASE64_STANDARD.encode(raw)) } } impl FromStr for Handwriting { type Err = eyre::Report; fn from_str(s: &str) -> Result { let s = try_from_custom_code_block(CODE_BLOCK_KEY, s) .ok_or_eyre("Not a valid ```handwriting-block")?; let bytes = BASE64_STANDARD .decode(s) .wrap_err("Failed to decode painting data from base64")?; // HACK: first iteration of disk format did not have version header //let mut bytes = bytes; //bytes.insert(0, 0); //bytes.insert(0, 1); let disk_format = DiskFormat::ref_from_bytes(&bytes[..]).map_err(|_| eyre!("Too short"))?; if disk_format.header.version != disk_format::V1 { bail!( "Unknown disk_format version: {}", disk_format.header.version ); } let mut raw_strokes = &disk_format.strokes[..]; let mut strokes = vec![]; while !raw_strokes.is_empty() { if raw_strokes.len() < RawStroke::MIN_LEN { bail!("Invalid remaining length: {}", raw_strokes.len()); } let stroke = RawStroke::ref_from_bytes(&raw_strokes[..RawStroke::MIN_LEN]) .expect("length is correct"); // get length as number of points let len = usize::from(u16::from(stroke.header.len)); // convert to length in bytes let byte_len = 2 * size_of::() * len; if raw_strokes.len() < byte_len { bail!("Invalid remaining length: {}", raw_strokes.len()); } let (stroke, rest) = raw_strokes.split_at(RawStroke::MIN_LEN + byte_len); raw_strokes = rest; let stroke = RawStroke::ref_from_bytes(stroke).expect("length is correct"); debug_assert_eq!( stroke.positions.len().rem_euclid(2), 0, "{} must be divisible by 2", stroke.positions.len() ); debug_assert_eq!(stroke.positions.len(), len * 2); let mut last_pos = Pos2::new(f32::NEG_INFINITY, f32::INFINITY); // positions are encoded as an array of f16s [x, y, x, y, x, y, ..] let stroke: Vec = stroke .positions .chunks_exact(2) .map(|chunk| [chunk[0], chunk[1]]) .map(|pos| pos.map(f16::from)) // interpret bytes as f16 .map(|pos| pos.map(f32::from)) // widen to f32 .filter(|pos| pos.iter().all(|f| f.is_finite())) // filter out NaNs and Infs .map(|[x, y]| Pos2::new(x, y)) .filter(|pos| { let is_duplicate = pos == &last_pos; last_pos = *pos; !is_duplicate // skip duplicates }) .collect(); strokes.push(stroke); } Ok(Handwriting { strokes, ..Default::default() }) } } impl HandwritingStyle { pub fn from_theme(theme: Theme) -> Self { let stroke_color; let bg_color; let line_color; match theme { Theme::Dark => { stroke_color = Color32::WHITE; bg_color = Color32::from_gray(30); line_color = Color32::from_rgb(100, 100, 100); } Theme::Light => { stroke_color = Color32::BLACK; bg_color = Color32::WHITE; line_color = Color32::from_rgb(130, 130, 130); // TODO } } HandwritingStyle { stroke: Stroke::new(1.0, stroke_color), bg_color, bg_line_stroke: Stroke::new(0.5, line_color), animate: true, hide_cursor: false, } } } fn mesh_triangles(mesh: &Mesh) -> impl Iterator + Clone { mesh.indices .chunks_exact(3) .map(|chunk| [chunk[0], chunk[1], chunk[2]]) .map(|indices| indices.map(|i| &mesh.vertices[i as usize])) } #[cfg(test)] mod test { use std::str::FromStr; use super::Handwriting; #[test] fn serialize_handwriting() { let handwriting = Handwriting::example(); insta::assert_debug_snapshot!("handwriting example", handwriting.strokes); let serialized = handwriting.to_string(); insta::assert_snapshot!("serialized handwriting", serialized); let deserialized = Handwriting::from_str(&serialized).expect("Handwriting must de/serialize correctly"); insta::assert_debug_snapshot!("deserialized handwriting", deserialized.strokes); } }