From f1e5f7bfa80d3b27e338214315375c1d04fbeb2d Mon Sep 17 00:00:00 2001 From: Simon Repp Date: Fri, 12 Apr 2024 19:03:53 +0200 Subject: [PATCH] Introduce deployment config fields, implement config probe and arm deployment --- Cargo.lock | 72 ++++++++++++++++++ Cargo.toml | 1 + src/build/widgets.rs | 2 +- src/deploy.rs | 140 +++++++++++++++++++++++++++++++---- src/editor/endpoints/site.rs | 81 +++++++++++++++++--- src/editor/widgets.rs | 9 ++- src/main.rs | 2 +- src/site.rs | 105 +++++++++++++++++++++++++- 8 files changed, 383 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 286dd2b..8f9d44e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1137,6 +1137,7 @@ dependencies = [ "image", "indoc", "language-tags", + "minreq", "nanoid", "notify", "pulldown-cmark", @@ -1511,6 +1512,19 @@ dependencies = [ "adler", ] +[[package]] +name = "minreq" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00a000cf8bbbfb123a9bdc66b61c2885a4bb038df4f2629884caafabeb76b0f9" +dependencies = [ + "log", + "once_cell", + "rustls", + "rustls-webpki", + "webpki-roots", +] + [[package]] name = "mio" version = "0.8.11" @@ -2009,6 +2023,20 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "ring" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babe80d5c16becf6594aa32ad2be8fe08498e7ae60b77de8df700e67f191d7e" +dependencies = [ + "cc", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.48.0", +] + [[package]] name = "rsubs-lib" version = "0.1.10" @@ -2047,6 +2075,28 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.21.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "ryu" version = "1.0.12" @@ -2068,6 +2118,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "semver" version = "1.0.16" @@ -2506,6 +2566,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.0" @@ -2649,6 +2715,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + [[package]] name = "weezl" version = "0.1.8" diff --git a/Cargo.toml b/Cargo.toml index 4f31259..08630cb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ futures-util = "0.3.30" image = "0.25.0" indoc = "2.0.5" language-tags = "0.3.2" +minreq = { features = ["https-rustls"], version = "2.11.1" } nanoid = "0.4.0" notify = "6.1.1" rsubs-lib = "0.1.10" diff --git a/src/build/widgets.rs b/src/build/widgets.rs index 36fea70..b5afa9a 100644 --- a/src/build/widgets.rs +++ b/src/build/widgets.rs @@ -40,7 +40,7 @@ fn compute_representations(versions: &[VideoFile]) -> Vec { pub fn copy_link(site: &Site, path: &str) -> String { let data_url = match &site.base_url { Some(base_url) => { - let url = base_url.join(path).unwrap().to_string(); + let url = base_url.value.join(path).unwrap().to_string(); format!(r#"data-url="{url}""#) } None => String::new() diff --git a/src/deploy.rs b/src/deploy.rs index 4b672da..3225c14 100644 --- a/src/deploy.rs +++ b/src/deploy.rs @@ -1,28 +1,140 @@ +use std::fs; +use std::path::Path; use std::process::Command; -use crate::Context; +use crate::{Context, Site}; +use crate::util::nanoid; + +const PROBE_FILE_NAME: &str = ".hyper8_config_probe.txt"; +const PROBE_TEXT: &str = "\ +This file was created by Hyper 8 to verify the configured server and path\ +for deployment matches up with the configured website url. It serves no\ +further purpose anymore and may safely be deleted.\ +"; /// Windows = Dummy implementation - we need a packaged rsync for this, which is not so trivial pub const RSYNC: &str = if cfg!(windows) { "rsync.exe" } else { "rsync" }; +/// Uploads a file named ".hyper8_config_probe.txt", which also contains a random token, +/// to the remote, thereby first verifying that the deployment configuration is actually +/// correct and working. Then, right next, tries to fetch that same file through the public +/// url of the website (e.g. https://example.com/.hyper8_config_probe.txt), thereby confirming +/// that the website is set up correctly (although that is a side effect), but more importantly +/// ensuring that no other website (or remote content of any kind) is accidentally wiped if the +/// remote path has been accidentally set incorrectly. If the probe fails in any way, the probe +/// file is removed from the remote again, leaving a clean state. If the probe succeeds the probe +/// file is left, as it is anyway removed through the consecutive deployment operation. +fn config_probe(context: &Context, site: &Site) -> Result<(), String> { + let probe_signature = nanoid(); + let probe_content = format!("{PROBE_TEXT}\n\n{probe_signature}"); + + let local_probe_path = context.build_dir.join(&PROBE_FILE_NAME); + let remote_probe_path = format!( + "{user}@{server}:{path}", + path = Path::new(site.deploy_config.path.as_ref().unwrap()).join(PROBE_FILE_NAME).to_string_lossy(), + server = site.deploy_config.server.as_ref().unwrap(), + user = site.deploy_config.user.as_ref().unwrap() + ); + + fs::write(&local_probe_path, &probe_content).unwrap(); + + let mut upload_probe_command = Command::new(RSYNC); + + upload_probe_command + .arg(&local_probe_path) + .arg(&remote_probe_path); + + let upload_probe_result = upload_probe_command.output(); + + let _ = fs::remove_file(&local_probe_path); + + match upload_probe_result { + Ok(output) => { + if !output.status.success() { + return Err(format!("The rsync child process for syncing the config probe to the remote errored. ({output:?})")) + } + } + Err(err) => return Err(format!("The rsync child process for syncing the config probe to the remote could not be executed. ({err})")) + } + + let public_probe_url = site.base_url.as_ref().unwrap().value.join(PROBE_FILE_NAME).unwrap(); + + match minreq::get(public_probe_url.clone()).send() { + Ok(response) => { + if response.as_str().is_ok_and(|body| body == probe_content) { + // Note that we are not removing the probe on the remote here - we already removed it locally + // in the build dir and the following deploy operation will take care of the removal on the remote. + Ok(()) + } else { + let remote_deploy_path = format!( + "{user}@{server}:{path}", + path = Path::new(site.deploy_config.path.as_ref().unwrap()).join("./").to_string_lossy(), + server = site.deploy_config.server.as_ref().unwrap(), + user = site.deploy_config.user.as_ref().unwrap() + ); + + let mut remove_probe_command = Command::new(RSYNC); + + // We're creatively combining rsync's options here to sync a single file to + // the remote which does not exist locally anymore, it therefore is removed. + remove_probe_command + .arg("--delete") + .arg("--dirs") + .arg(&format!("--include={PROBE_FILE_NAME}")) // must appear before --exclude (!) otherwise removal does not happen + .arg("--exclude=*") + .arg(&context.build_dir.join("./")) + .arg(&remote_deploy_path); + + let _ = remove_probe_command.output(); + + Err(format!("The config probe was successfully uploaded to {remote_probe_path} but its presence could not be verified through the public interface {public_probe_url}. (Double-check that both the base url and the deployment configuration are correct)")) + } + } + Err(err) => return Err(format!("The config probe was successfully uploaded to {remote_probe_path} but not obtainable through the public interface {public_probe_url}. ({err})")) + } +} + pub fn perform_deploy(context: &Context) { + let site = context.get_site(); + + // TODO: All of these errors (currently: panics) should be gracefully communicated through the browser interface + + if !site.base_url.is_some() { + panic!("Base url is not set, therefore the deploy config cannot be probed, therefore deploying is not permitted for safety reasons."); + } + + if !site.deploy_config.complete() { + panic!("Deploy config is not complete, can not deploy."); + } + + if let Err(err) = config_probe(context, &site) { + panic!("The deployment configuration could not be verified: {err}"); + } + + let remote_deploy_path = format!( + "{user}@{server}:{path}", + path = Path::new(site.deploy_config.path.as_ref().unwrap()).join("./").to_string_lossy(), + server = site.deploy_config.server.as_ref().unwrap(), + user = site.deploy_config.user.as_ref().unwrap() + ); + let mut command = Command::new(RSYNC); command.arg("-avP"); command.arg("--delete"); command.arg(context.build_dir.join("./")); - command.arg("deploy/"); // TODO: Local dummy target for now + command.arg(remote_deploy_path); - // TODO: Disarmed for now - // match command.output() { - // Ok(output) => { - // if !output.status.success() { - // eprint!("{output:?}"); - // } - // } - // Err(err) => { - // eprint!("The rsync child process could not be executed."); - // eprint!("{err}"); - // } - // } + // TODO: Pass errors back to interface and user + match command.output() { + Ok(output) => { + if !output.status.success() { + eprint!("{output:?}"); + } + } + Err(err) => { + eprint!("The rsync child process could not be executed."); + eprint!("{err}"); + } + } } \ No newline at end of file diff --git a/src/editor/endpoints/site.rs b/src/editor/endpoints/site.rs index ec9b40b..d9e7f00 100644 --- a/src/editor/endpoints/site.rs +++ b/src/editor/endpoints/site.rs @@ -4,10 +4,10 @@ use actix_web::HttpResponse; use actix_web::web::{Data, Either, Form, Redirect}; use indoc::formatdoc; use serde_derive::Deserialize; -use url::Url; use crate::{ AspectRatio, + BaseUrl, Context, Field, Language, @@ -18,9 +18,19 @@ use crate::{ use crate::editor::endpoints; use crate::editor::widgets::form_field; +const LABEL_BASE_URL: &str = "Base URL (e.g. https://example.com)"; +const LABEL_DEPLOYMENT_PATH: &str = "Deployment Path (e.g. /data/web/customer123/html"; +const LABEL_DEPLOYMENT_SERVER: &str = "Deployment Server (e.g. https://ssh-customer123.example.com)"; +const LABEL_DEPLOYMENT_USER: &str = "Deployment User (e.g. customer123)"; +const LABEL_LANGUAGE: &str = "Language (e.g. en, fr)"; +const LABEL_POSTER_ASPECT: &str = "Custom Poster Aspect Ratio (e.g. 16/9, 4:3, 1.778)"; + #[derive(Deserialize)] pub struct SiteForm { base_url: String, + deployment_path: String, + deployment_server: String, + deployment_user: String, language: String, poster_aspect: String, theme: String @@ -28,6 +38,9 @@ pub struct SiteForm { pub struct SiteFormFeedback { pub base_url: Field, + pub deployment_path: Field, + pub deployment_server: Field, + pub deployment_user: Field, pub language: Field, pub poster_aspect: Field, pub theme: Theme @@ -129,18 +142,35 @@ fn edit_page( // // + // TODO: For better usability, include detailed hints (help texts, possibly toggeable, hidden by default) + // that give very clear indication about what base url, deployment path/server/user etc. are and + // what to pay attention to. let ( theme_key, input_base_url, + input_deployment_path, + input_deployment_server, + input_deployment_user, input_language, input_poster_aspect, feedback - ) = if let Some(SiteFormFeedback { base_url, language, poster_aspect, theme }) = &form_feedback { + ) = if let Some(SiteFormFeedback { + base_url, + deployment_path, + deployment_server, + deployment_user, + language, + poster_aspect, + theme + }) = &form_feedback { ( &theme.key, - form_field(base_url, "base_url", "Base URL (e.g. https://example.com)"), - form_field(language, "language", "Language (e.g. en, fr)"), - form_field(poster_aspect, "poster_aspect", "Custom Poster Aspect Ratio (e.g. 16/9, 4:3, 1.778)"), + form_field(base_url, "base_url", LABEL_BASE_URL), + form_field(deployment_path, "deployment_path", LABEL_DEPLOYMENT_PATH), + form_field(deployment_server, "deployment_server", LABEL_DEPLOYMENT_SERVER), + form_field(deployment_user, "deployment_user", LABEL_DEPLOYMENT_USER), + form_field(language, "language", LABEL_LANGUAGE), + form_field(poster_aspect, "poster_aspect", LABEL_POSTER_ASPECT), r#""# ) } else { @@ -156,9 +186,12 @@ fn edit_page( ( &site.theme.key, - form_field(&Field::valid(site.base_url.as_ref().map(|url| url.as_str()).unwrap_or("")), "base_url", "Base URL (e.g. https://example.com)"), - form_field(&Field::valid(language), "language", "Language (e.g. en, fr)"), - form_field(&Field::valid(poster_aspect), "poster_aspect", "Custom Poster Aspect Ratio (e.g. 16/9, 4:3, 1.778)"), + form_field(&Field::valid(site.base_url.as_ref().map(|url| url.representation.as_str()).unwrap_or("")), "base_url", LABEL_BASE_URL), + form_field(&Field::valid(site.deploy_config.path.as_ref().map(|path| path.as_str()).unwrap_or("")), "deployment_path", LABEL_DEPLOYMENT_PATH), + form_field(&Field::valid(site.deploy_config.server.as_ref().map(|server| server.as_str()).unwrap_or("")), "deployment_server", LABEL_DEPLOYMENT_SERVER), + form_field(&Field::valid(site.deploy_config.user.as_ref().map(|user| user.as_str()).unwrap_or("")), "deployment_user", LABEL_DEPLOYMENT_USER), + form_field(&Field::valid(language), "language", LABEL_LANGUAGE), + form_field(&Field::valid(poster_aspect), "poster_aspect", LABEL_POSTER_ASPECT), "" ) }; @@ -184,6 +217,9 @@ fn edit_page(
{input_base_url} + {input_deployment_path} + {input_deployment_server} + {input_deployment_user} {input_poster_aspect} {input_language}
@@ -239,6 +275,9 @@ pub async fn update( form: Form ) -> Either { let base_url_trimmed = form.base_url.trim(); + let deployment_path_trimmed = form.deployment_path.trim(); + let deployment_server_trimmed = form.deployment_server.trim(); + let deployment_user_trimmed = form.deployment_user.trim(); let language_trimmed = form.language.trim(); let poster_aspect_trimmed = form.poster_aspect.trim(); let theme = match Theme::from_key(&form.theme) { @@ -249,7 +288,7 @@ pub async fn update( let base_url = if base_url_trimmed.is_empty() { None } else { - Some(Url::parse(base_url_trimmed)) + Some(BaseUrl::parse(&base_url_trimmed)) }; let poster_aspect = if poster_aspect_trimmed.is_empty() { @@ -264,6 +303,9 @@ pub async fn update( let site_form_feedback = SiteFormFeedback { base_url: Field::validated(base_url_trimmed, base_url.and_then(|base_url| base_url.err().map(|err| err.to_string()))), + deployment_path: Field::valid(deployment_path_trimmed), + deployment_server: Field::valid(deployment_server_trimmed), + deployment_user: Field::valid(deployment_user_trimmed), language: Field::valid(language_trimmed), poster_aspect: Field::validated(poster_aspect_trimmed, poster_aspect.err()), theme @@ -288,6 +330,24 @@ pub async fn update( Either::Right(http_response) } else { + let deployment_path = if deployment_path_trimmed.is_empty() { + None + } else { + Some(deployment_path_trimmed.to_string()) + }; + + let deployment_server = if deployment_server_trimmed.is_empty() { + None + } else { + Some(deployment_server_trimmed.to_string()) + }; + + let deployment_user = if deployment_user_trimmed.is_empty() { + None + } else { + Some(deployment_user_trimmed.to_string()) + }; + let language = if language_trimmed.is_empty() { Language::default() } else { @@ -296,6 +356,9 @@ pub async fn update( let mutation = |site_mut: &mut Site| { site_mut.base_url = base_url.map(|base_url| base_url.unwrap()); + site_mut.deploy_config.path = deployment_path; + site_mut.deploy_config.server = deployment_server; + site_mut.deploy_config.user = deployment_user; site_mut.language = language; site_mut.poster_aspect = poster_aspect.unwrap(); site_mut.theme = theme; diff --git a/src/editor/widgets.rs b/src/editor/widgets.rs index 40778c4..33b9af8 100644 --- a/src/editor/widgets.rs +++ b/src/editor/widgets.rs @@ -297,10 +297,17 @@ pub fn layout( let body = formatdoc!(r#" + let deploy_button = match site.deploy_config.complete() { + true => r#""#, + // TODO: Instead of disabled link, offer "Configure Deployment" link + // (Site form + hash that goes directly to the relevant section, or dedicated page to clean up interface by area of concern?) + false => r#""# + }; +
-
+ {deploy_button}
Hyper 8 Video System {hyper8_version} diff --git a/src/main.rs b/src/main.rs index dfb2db4..826292d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -40,7 +40,7 @@ use language::Language; use playlist::Playlist; use poster::{PosterAsset, PosterAssets, PosterFile}; use processing::{Job, JobKind, Processing}; -use site::{AspectRatio, Site, SiteContent}; +use site::{AspectRatio, BaseUrl, Site, SiteContent}; use subtitles::SubtitleFile; use theme::{Theme, ThemeKind}; use util::Field; diff --git a/src/site.rs b/src/site.rs index 33212ee..087ca84 100644 --- a/src/site.rs +++ b/src/site.rs @@ -25,10 +25,29 @@ pub enum AspectRatio { Default } +/// Base url can be user-provided with or without trailing slash +/// ("https://example.com/subdir" "https://example.com/subdir/"), but for +/// further url construction through the url crate the trailing slash is +/// significant (https://docs.rs/url/latest/url/struct.Url.html#method.join), +/// hence we store both representation and a working copy of the parsed url. +#[derive(Clone, Debug)] +pub struct BaseUrl { + pub representation: String, + pub value: Url +} + +#[derive(Clone, Debug)] +pub struct DeployConfig { + pub path: Option, + pub server: Option, + pub user: Option +} + #[derive(Clone, Debug)] pub struct Site { - pub base_url: Option, + pub base_url: Option, pub content: SiteContent, + pub deploy_config: DeployConfig, pub errors: Vec, pub language: Language, pub poster_aspect: AspectRatio, @@ -79,6 +98,41 @@ impl AspectRatio { } } +impl BaseUrl { + /// Delegates parsing to the url crate but beforehand ensures a trailing + /// slash so further url construction is performed correctly. + pub fn parse(url: &str) -> Result { + let parsed_url = match url.ends_with('/') { + true => Url::parse(url), + false => Url::parse(&format!("{url}/")) + }; + + match parsed_url { + Ok(value) => { + Ok(BaseUrl { + representation: url.to_owned(), + value + }) + } + Err(err) => Err(err.to_string()) + } + } +} + +impl DeployConfig { + pub fn complete(&self) -> bool { + self.path.is_some() && self.server.is_some() && self.user.is_some() + } + + pub fn new() -> DeployConfig { + DeployConfig { + path: None, + server: None, + user: None + } + } +} + impl Site { fn apply_manifest(&mut self, manifest_path: &Path) { let content = fs::read_to_string(manifest_path).unwrap(); @@ -94,7 +148,7 @@ impl Site { match document.optional_field("base_url") { Ok(Some(field)) => { match field.required_value::() { - Ok(value) => match Url::parse(&value) { + Ok(value) => match BaseUrl::parse(&value) { Ok(url) => self.base_url = Some(url), Err(err) => self.errors.push(format!("Error in {}:{} ({})", manifest_path.display(), field.line_number(), err)) } @@ -105,6 +159,33 @@ impl Site { _ => () } + match document.optional_field("deploy_config") { + Ok(Some(field)) => match field.attributes() { + Ok(attributes) => { + for attribute in attributes { + match attribute.key() { + "path" => match attribute.required_value::() { + Ok(path) => self.deploy_config.path = Some(path), + Err(err) => self.errors.push(format!("Error in {}:{} ({})", manifest_path.display(), err.line, err.message)) + } + "server" => match attribute.required_value::() { + Ok(server) => self.deploy_config.server = Some(server), + Err(err) => self.errors.push(format!("Error in {}:{} ({})", manifest_path.display(), err.line, err.message)) + } + "user" => match attribute.required_value::() { + Ok(user) => self.deploy_config.user = Some(user), + Err(err) => self.errors.push(format!("Error in {}:{} ({})", manifest_path.display(), err.line, err.message)) + } + _ => self.errors.push(format!("Ignoring unsupported attribute '{}' in {}:{}", attribute.key(), manifest_path.display(), attribute.line_number())) + } + } + } + Err(err) => self.errors.push(format!("Error in {}:{} ({err})", manifest_path.display(), field.line_number())) + } + Err(err) => self.errors.push(format!("Error in {}:{} ({})", manifest_path.display(), err.line, err)), + _ => () + } + match document.optional_field("language") { Ok(Some(field)) => { match field.required_value::() { @@ -232,6 +313,7 @@ impl Site { Site { base_url: None, content: SiteContent::Empty, + deploy_config: DeployConfig::new(), errors: Vec::new(), language: Language::default(), poster_aspect: AspectRatio::Default, @@ -296,7 +378,24 @@ impl Site { let mut eno = String::new(); if let Some(base_url) = &self.base_url { - eno.push_str(&format!("base_url: {}\n", base_url)); + eno.push_str(&format!("base_url: {}\n", base_url.representation)); + } + + if self.deploy_config.server.is_some() || + self.deploy_config.user.is_some() { + eno.push_str("deploy_config:\n"); + + if let Some(path) = &self.deploy_config.path { + eno.push_str(&format!("path = {}\n", path)); + } + + if let Some(server) = &self.deploy_config.server { + eno.push_str(&format!("server = {}\n", server)); + } + + if let Some(user) = &self.deploy_config.user { + eno.push_str(&format!("user = {}\n", user)); + } } if !self.language.default {