From bcaade071f1ad97317447bdfb0833ad264cca87d Mon Sep 17 00:00:00 2001 From: Eric Van Albert Date: Thu, 29 Jun 2023 23:13:57 -0400 Subject: [PATCH] Add bouncing ball widget --- src/bin/main.rs | 17 ++- src/bin/ui/beat_widget.rs | 221 ++++++++++++++++++++++++++++++++++ src/bin/ui/beat_widget.wgsl | 67 +++++++++++ src/bin/ui/mod.rs | 2 + src/bin/ui/spectrum_widget.rs | 1 - src/bin/ui/waveform_widget.rs | 1 - src/lib/mir.rs | 25 ++-- 7 files changed, 323 insertions(+), 11 deletions(-) create mode 100644 src/bin/ui/beat_widget.rs create mode 100644 src/bin/ui/beat_widget.wgsl diff --git a/src/bin/main.rs b/src/bin/main.rs index 19b67019..0f4443bb 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -28,7 +28,7 @@ use radiance::{ mod ui; use ui::{modal, modal_shown, mosaic}; -use ui::{SpectrumWidget, WaveformWidget}; +use ui::{BeatWidget, SpectrumWidget, WaveformWidget}; mod winit_output; use winit_output::WinitOutput; @@ -171,6 +171,7 @@ pub async fn run() { // Make widgets let mut waveform_widget = WaveformWidget::new(device.clone(), queue.clone(), pixels_per_point); let mut spectrum_widget = SpectrumWidget::new(device.clone(), queue.clone(), pixels_per_point); + let mut beat_widget = BeatWidget::new(device.clone(), queue.clone(), pixels_per_point); // Make an AutoDJ let mut auto_dj_1: Option = None; @@ -336,6 +337,7 @@ pub async fn run() { let mut waveform_texture: Option = None; let mut spectrum_texture: Option = None; + let mut beat_texture: Option = None; let mut autosave_timer: usize = 0; @@ -438,7 +440,7 @@ pub async fn run() { let waveform_native_texture = waveform_widget.paint( waveform_size, &music_info.audio, - music_info.uncompensated_time, + music_info.uncompensated_unscaled_time, ); update_or_register_native_texture( @@ -459,6 +461,16 @@ pub async fn run() { &mut spectrum_texture, ); + let beat_size = egui::vec2(65., 65.); + let beat_native_texture = beat_widget.paint(beat_size, music_info.unscaled_time); + + update_or_register_native_texture( + &mut egui_renderer, + &device, + &beat_native_texture.view, + &mut beat_texture, + ); + // EGUI update let raw_input = platform.take_egui_input(&window); let full_output = egui_ctx.run(raw_input, |egui_ctx| { @@ -478,6 +490,7 @@ pub async fn run() { ui.horizontal(|ui| { ui.image(waveform_texture.unwrap(), waveform_size); ui.image(spectrum_texture.unwrap(), spectrum_size); + ui.image(beat_texture.unwrap(), beat_size); ui.checkbox(&mut auto_dj_1_enabled, "Auto DJ 1"); ui.checkbox(&mut auto_dj_2_enabled, "Auto DJ 2"); diff --git a/src/bin/ui/beat_widget.rs b/src/bin/ui/beat_widget.rs new file mode 100644 index 00000000..abba27f7 --- /dev/null +++ b/src/bin/ui/beat_widget.rs @@ -0,0 +1,221 @@ +use egui::Vec2; +use radiance::ArcTextureViewSampler; +use std::iter; +use std::sync::Arc; + +pub struct BeatWidget { + // Constructor arguments: + device: Arc, + queue: Arc, + pixels_per_point: f32, + + // Internal state: + width: u32, + height: u32, + texture: ArcTextureViewSampler, + _shader_module: wgpu::ShaderModule, + uniform_buffer: wgpu::Buffer, + bind_group: wgpu::BindGroup, + render_pipeline: wgpu::RenderPipeline, +} + +// The uniform buffer associated with the ball +#[repr(C)] +#[derive(Default, Debug, Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)] +struct Uniforms { + resolution: [f32; 2], // in pixels + size: [f32; 2], // in points + beat: f32, + _padding: [u8; 4], +} + +impl BeatWidget { + fn make_texture(device: &wgpu::Device, width: u32, height: u32) -> ArcTextureViewSampler { + let texture_size = wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }; + + let texture_desc = wgpu::TextureDescriptor { + size: texture_size, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::COPY_SRC + | wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING, + label: None, + }; + + let texture = device.create_texture(&texture_desc); + + let view = texture.create_view(&Default::default()); + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + address_mode_w: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + mipmap_filter: wgpu::FilterMode::Linear, + ..Default::default() + }); + ArcTextureViewSampler::new(texture, view, sampler) + } + + pub fn new(device: Arc, queue: Arc, pixels_per_point: f32) -> Self { + let width = 1; + let height = 1; + let texture = Self::make_texture(&device, width, height); + + let shader_module = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some(&"beat widget shader module"), + source: wgpu::ShaderSource::Wgsl(include_str!("beat_widget.wgsl").into()), + }); + + // The uniform buffer for this widget + let uniform_buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some(&"beat widget uniform buffer"), + size: std::mem::size_of::() as u64, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some(&"beat widget bind group layout"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, // Uniforms + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }], + }); + + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + layout: &bind_group_layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: uniform_buffer.as_entire_binding(), + }], + label: Some("beat bind group"), + }); + + let render_pipeline_layout = + device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("Beat widget render pipeline layout"), + bind_group_layouts: &[&bind_group_layout], + push_constant_ranges: &[], + }); + + let render_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("Beat widget render pipeline"), + layout: Some(&render_pipeline_layout), + vertex: wgpu::VertexState { + module: &shader_module, + entry_point: "vs_main", + buffers: &[], + }, + fragment: Some(wgpu::FragmentState { + module: &shader_module, + entry_point: "fs_main", + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: Some(wgpu::BlendState::REPLACE), + write_mask: wgpu::ColorWrites::ALL, + })], + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleStrip, + strip_index_format: None, + front_face: wgpu::FrontFace::Ccw, + cull_mode: Some(wgpu::Face::Back), + polygon_mode: wgpu::PolygonMode::Fill, + unclipped_depth: false, + conservative: false, + }, + depth_stencil: None, + multisample: wgpu::MultisampleState { + count: 1, + mask: !0, + alpha_to_coverage_enabled: false, + }, + multiview: None, + }); + + Self { + device, + queue, + pixels_per_point, + + width, + height, + texture, + _shader_module: shader_module, + bind_group, + uniform_buffer, + render_pipeline, + } + } + + pub fn paint(&mut self, size: Vec2, beat: f32) -> ArcTextureViewSampler { + // Possibly remake the texture if the size has changed + let width = (size.x * self.pixels_per_point) as u32; + let height = (size.y * self.pixels_per_point) as u32; + if width != self.width || height != self.height { + self.width = width; + self.height = height; + self.texture = Self::make_texture(&self.device, width, height); + } + + // Populate the uniforms + let uniforms = Uniforms { + resolution: [width as f32, height as f32], + size: [size.x as f32, size.y as f32], + beat, + ..Default::default() + }; + + self.queue + .write_buffer(&self.uniform_buffer, 0, bytemuck::cast_slice(&[uniforms])); + + let mut encoder = self + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("Beat widget encoder"), + }); + // Record output render pass. + { + let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("Output window render pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.texture.view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color { + r: 0., + g: 0., + b: 0., + a: 0., + }), + store: true, + }, + })], + depth_stencil_attachment: None, + }); + + render_pass.set_pipeline(&self.render_pipeline); + render_pass.set_bind_group(0, &self.bind_group, &[]); + render_pass.draw(0..4, 0..1); + } + + // Submit the commands. + self.queue.submit(iter::once(encoder.finish())); + + self.texture.clone() + } +} diff --git a/src/bin/ui/beat_widget.wgsl b/src/bin/ui/beat_widget.wgsl new file mode 100644 index 00000000..f6557269 --- /dev/null +++ b/src/bin/ui/beat_widget.wgsl @@ -0,0 +1,67 @@ +struct Uniforms { + resolution: vec2, + size: vec2, + beat: f32, +} + +@group(0) @binding(0) +var global: Uniforms; + +struct VertexOutput { + @builtin(position) gl_Position: vec4, + @location(0) uv: vec2, +}; + +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { + var pos_array = array, 4>( + vec2(1., 1.), + vec2(-1., 1.), + vec2(1., -1.), + vec2(-1., -1.), + ); + var uv_array = array, 4>( + vec2(1., 0.), + vec2(0., 0.), + vec2(1., 1.), + vec2(0., 1.), + ); + + return VertexOutput( + vec4(pos_array[vertex_index], 0., 1.), + uv_array[vertex_index], + ); +} + +// Alpha-compsite two colors, putting one on top of the other +fn composite(under: vec4, over: vec4) -> vec4 { + let a_out = 1. - (1. - over.a) * (1. - under.a); + return clamp(vec4((over.rgb + under.rgb * (1. - over.a)), a_out), vec4(0.), vec4(1.)); +} + +// Box from [0, 0] to (1, 1) +fn box(p: vec2) -> f32 { + let b = step(vec2(0.), p) - step(vec2(1.), p); + return b.x * b.y; +} + +@fragment +fn fs_main(vertex: VertexOutput) -> @location(0) vec4 { + let ballOutlineColor = vec4(0.267, 0., 0.444, 1.); + let ballColorBottom = vec4(0.4, 0., 0.667, 1.); + let ballColorTop = vec4(0.667, 0., 1., 1.); + let floorColor = vec4(0.667, 0.667, 0.667, 1.); + + let height = 1. - pow(abs(2. * (fract(global.beat) - 0.5)), 2.); + let height = height * (1. - 0.4 * (1. - step(3., global.beat % 4.))); + let ballLoc = vec2(0.5, 0.8 - 0.6 * height); + let ballColor = mix(ballColorBottom, ballColorTop, clamp(10. * (ballLoc.y - vertex.uv.y), 0., 1.)); + let ball = 1. - smoothstep(0.08, 0.09, length(vertex.uv - ballLoc)); + let ballOutline = 1. - smoothstep(0.09, 0.1, length(vertex.uv - ballLoc)); + let floorBox = box((vertex.uv - vec2(0.2, 0.9)) / vec2(0.6, 0.05)); + + let fragColor = floorBox * floorColor; + let fragColor = composite(fragColor, ballOutline * ballOutlineColor); + let fragColor = composite(fragColor, ball * ballColor); + return fragColor; +} diff --git a/src/bin/ui/mod.rs b/src/bin/ui/mod.rs index 3766fa7e..b1bd6679 100644 --- a/src/bin/ui/mod.rs +++ b/src/bin/ui/mod.rs @@ -1,3 +1,4 @@ +mod beat_widget; mod drop_target; mod effect_node_tile; mod image_node_tile; @@ -11,6 +12,7 @@ mod spectrum_widget; mod tile; mod waveform_widget; +pub use beat_widget::*; pub use drop_target::*; pub use effect_node_tile::*; pub use image_node_tile::*; diff --git a/src/bin/ui/spectrum_widget.rs b/src/bin/ui/spectrum_widget.rs index e9ef4c1a..a7c9bf39 100644 --- a/src/bin/ui/spectrum_widget.rs +++ b/src/bin/ui/spectrum_widget.rs @@ -26,7 +26,6 @@ pub struct SpectrumWidget { struct Uniforms { resolution: [f32; 2], // in pixels size: [f32; 2], // in points - _padding: [u8; 8], } #[repr(C)] diff --git a/src/bin/ui/waveform_widget.rs b/src/bin/ui/waveform_widget.rs index dff3ece0..a6e8a0d5 100644 --- a/src/bin/ui/waveform_widget.rs +++ b/src/bin/ui/waveform_widget.rs @@ -32,7 +32,6 @@ pub struct WaveformWidget { struct Uniforms { resolution: [f32; 2], // in pixels size: [f32; 2], // in points - _padding: [u8; 8], } #[repr(C)] diff --git a/src/lib/mir.rs b/src/lib/mir.rs index 68db4898..7eda3db6 100644 --- a/src/lib/mir.rs +++ b/src/lib/mir.rs @@ -6,7 +6,7 @@ use std::time; const MAX_TIME: f32 = 64.; // Anticipate beats by this many seconds -const LATENCY_COMPENSATION: f32 = 0.07; +const LATENCY_COMPENSATION: f32 = 0.10; const DEFAULT_BPM: f32 = 120.; @@ -68,9 +68,12 @@ impl Update { /// containing real-time information about the audio. #[derive(Clone, Debug)] pub struct MusicInfo { - pub time: f32, // time in beats - pub uncompensated_time: f32, // time in beats, without latency compensation (so it aligns with reported audio levels) - pub tempo: f32, // beats per second + pub time: f32, // time in beats + pub unscaled_time: f32, // time in beats, without the global timescale applied + // (for widgets that shouldn't be affected by global + // timescale) + pub uncompensated_unscaled_time: f32, // time in beats, without the global timescale or latency compensation (so it aligns with reported audio levels) + pub tempo: f32, // beats per second pub audio: AudioLevels, pub spectrum: [f32; SPECTRUM_LENGTH], } @@ -272,8 +275,15 @@ impl Mir { } // Compute t - let uncompensated_time = - (self.last_update.t(time::Instant::now()) * self.global_timescale).rem_euclid(MAX_TIME); + let uncompensated_unscaled_time = self + .last_update + .t(time::Instant::now()) + .rem_euclid(MAX_TIME); + + let unscaled_time = self + .last_update + .t(time::Instant::now() + time::Duration::from_secs_f32(LATENCY_COMPENSATION)) + .rem_euclid(MAX_TIME); let time = (self .last_update @@ -283,7 +293,8 @@ impl Mir { MusicInfo { time, - uncompensated_time, + unscaled_time, + uncompensated_unscaled_time, tempo: self.last_update.tempo * self.global_timescale, audio: self.last_update.audio.clone(), spectrum: self.last_update.spectrum.clone(),