CLI tool to upload media files and save them to the DB

Cargo.lock

@@ -1841,6 +1841,27 @@ dependencies = [
  "tokio 1.25.0",
 ]
 
+[[package]]
+name = "dev-upload-media-file"
+version = "0.0.1"
+dependencies = [
+ "buckets",
+ "clap",
+ "cloud_storage",
+ "config",
+ "easyenv",
+ "enums",
+ "errors",
+ "filesys",
+ "hashing",
+ "log",
+ "mimetypes",
+ "mysql_queries",
+ "sqlx",
+ "tokens",
+ "tokio 1.25.0",
+]
+
 [[package]]
 name = "diesel"
 version = "2.1.3"

Cargo.toml

@@ -12,6 +12,7 @@ members = [
   "crates/cli/database_backup",
   "crates/cli/database_cleanup",
   "crates/cli/dev_database_seed",
+  "crates/cli/dev_upload_media_file",
   "crates/cli/kubectl_pod_cleanup",
   "crates/cli/scrape_test",
 

crates/cli/dev_database_seed/src/seeding/users.rs

@@ -12,6 +12,7 @@ pub const HANASHI_USERNAME : &str = "hanashi";
 pub async fn seed_user_accounts(mysql_pool: &Pool<MySql>) -> AnyhowResult<()> {
   info!("Seeding user accounts...");
 
+  // NB: This is idempotent and will only install the accounts once.
   let users = [
     (ADMIN_USERNAME, "password"),
     (HANASHI_USERNAME, "password"),

crates/cli/dev_upload_media_file/Cargo.toml

@@ -0,0 +1,44 @@
+[package]
+name = "dev-upload-media-file"
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+publish = false
+
+[[bin]]
+name = "dev-upload-media-file"
+path = "src/main.rs"
+
+[dependencies]
+# Internal
+buckets = { path = "../../_types/buckets" }
+cloud_storage = { path = "../../lib/api_clients/cloud_storage" }
+config = { path = "../../lib/deprecated/config" }
+easyenv = { path = "../../lib/easyenv" }
+enums = { path = "../../_types/enums" }
+errors = { path = "../../lib/errors" }
+filesys = { path = "../../lib/files/filesys" }
+hashing = { path = "../../lib/files/hashing" }
+mimetypes = { path = "../../lib/files/mimetypes" }
+mysql_queries = { path = "../../lib/mysql_queries" }
+tokens = { path = "../../_types/tokens" }
+
+# External
+clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated
+log = "0.4.14"
+
+# Runtime
+# NB: We switched to Tokio from `async-std` for broader ecosystem support.
+tokio = { version = "1.6.0", features = ["macros"] }
+
+# MySql Database
+# "offline" supports typesafe builds without a live database
+# "rustls" doesn't appear to work with IP addresses, per:
+# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828
+sqlx = { version = "0.6.2", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] }
+
+[dev-dependencies]
+# None yet

crates/cli/dev_upload_media_file/src/cli_args.rs

@@ -0,0 +1,34 @@
+use std::path::PathBuf;
+use clap::{App, Arg};
+use errors::{anyhow, AnyhowResult};
+
+pub struct CliArgs {
+  pub file_path: PathBuf,
+}
+
+pub fn parse_cli_args() -> AnyhowResult<CliArgs> {
+  let matches = App::new("scrape_test")
+      .arg(Arg::with_name("file")
+          .short("f")
+          .long("file")
+          .value_name("FILE")
+          .help("File to upload")
+          .takes_value(true)
+          .required(true))
+      //.arg(Arg::with_name("print_html")
+      //  .long("html")
+      //  .help("Print the HTML scraped")
+      //  .takes_value(false)
+      //  .required(false))
+      .get_matches();
+
+  let file_path = matches.value_of("file")
+      .map(|val| val.trim().to_string())
+      .ok_or(anyhow!("no url supplied"))?;
+
+  //let print_html = matches.is_present("print_html");
+
+  Ok(CliArgs {
+    file_path: PathBuf::from(file_path),
+  })
+}

crates/cli/dev_upload_media_file/src/main.rs

@@ -0,0 +1,154 @@
+use std::time::Duration;
+
+use log::info;
+use sqlx::mysql::MySqlPoolOptions;
+
+use buckets::public::media_files::original_file::MediaFileBucketPath;
+use cloud_storage::bucket_client::BucketClient;
+use config::shared_constants::{DEFAULT_MYSQL_CONNECTION_STRING, DEFAULT_RUST_LOG};
+use enums::by_table::media_files::media_file_type::MediaFileType;
+use enums::common::visibility::Visibility;
+use errors::{AnyhowResult, bail};
+use filesys::file_read_bytes::file_read_bytes;
+use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
+use mysql_queries::queries::media_files::insert_media_file_from_cli_tool::{insert_media_file_from_cli_tool, InsertArgs};
+
+use crate::cli_args::parse_cli_args;
+
+pub mod cli_args;
+
+#[tokio::main]
+pub async fn main() -> AnyhowResult<()> {
+  easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG));
+
+  info!("Media file upload");
+
+  // NB: Read secrets (eg. ACCESS_KEY)
+  easyenv::from_filename(".env-secrets")?;
+
+  let db_connection_string =
+      easyenv::get_env_string_or_default(
+        "MYSQL_URL",
+        DEFAULT_MYSQL_CONNECTION_STRING);
+
+  info!("Connecting to MySQL...");
+
+  let pool = MySqlPoolOptions::new()
+      .max_connections(easyenv::get_env_num("MYSQL_MAX_CONNECTIONS", 3)?)
+      .connect(&db_connection_string)
+      .await?;
+
+  let access_key = easyenv::get_env_string_required("ACCESS_KEY")?;
+  let secret_key = easyenv::get_env_string_required("SECRET_KEY")?;
+  let region_name = easyenv::get_env_string_required("REGION_NAME")?;
+  let public_bucket_name = easyenv::get_env_string_required("PUBLIC_BUCKET_NAME")?;
+
+  let bucket_timeout = easyenv::get_env_duration_seconds_or_default(
+    "BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 5));
+
+  info!("Configuring GCS bucket...");
+
+  let public_bucket_client = BucketClient::create(
+    &access_key,
+    &secret_key,
+    &region_name,
+    &public_bucket_name,
+    None,
+    Some(bucket_timeout),
+  )?;
+
+  let args = parse_cli_args()?;
+
+  info!("Reading file to upload: {:?} ...", args.file_path);
+
+  let bytes = file_read_bytes(args.file_path)?;
+  let mimetype = get_mimetype_for_bytes(&bytes);
+
+  let mut media_file_type;
+  let mut maybe_mime_type;
+  let mut maybe_public_bucket_prefix;
+  let mut maybe_public_bucket_extension;
+
+  match mimetype {
+    Some("audio/wav") |
+    Some("audio/x-wav") => {
+      media_file_type = MediaFileType::Video;
+      maybe_mime_type = Some("audio/wav");
+      maybe_public_bucket_prefix = Some("dev_upload_");
+      maybe_public_bucket_extension = Some(".wav");
+
+    }
+    Some("video/mp4") => {
+      media_file_type = MediaFileType::Video;
+      maybe_mime_type = mimetype;
+      maybe_public_bucket_prefix = Some("dev_upload_");
+      maybe_public_bucket_extension = Some(".mp4");
+    }
+    Some("image/jpeg") => {
+      media_file_type = MediaFileType::Image;
+      maybe_mime_type = mimetype;
+      maybe_public_bucket_prefix = Some("dev_upload_");
+      maybe_public_bucket_extension = Some(".jpg");
+    }
+    Some("image/png") => {
+      media_file_type = MediaFileType::Image;
+      maybe_mime_type = mimetype;
+      maybe_public_bucket_prefix = Some("dev_upload_");
+      maybe_public_bucket_extension = Some(".png");
+    }
+    _ => {
+      bail!("Invalid mime type: {:?}", mimetype);
+    }
+  }
+
+  let mimetype = mimetype.unwrap_or("text/plain");
+
+  let creator_set_visibility = Visibility::Public;
+
+  let public_upload_path = MediaFileBucketPath::generate_new(
+    maybe_public_bucket_prefix,
+    maybe_public_bucket_extension);
+
+  info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str());
+
+  let _r = public_bucket_client.upload_file_with_content_type(
+    public_upload_path.get_full_object_path_str(),
+    bytes.as_ref(),
+    mimetype)
+      .await?;
+
+  let (media_file_token, _id) = insert_media_file_from_cli_tool(InsertArgs {
+    pool: &pool,
+    media_file_type,
+    maybe_mime_type,
+    file_size_bytes: 0,
+    sha256_checksum: "",
+    maybe_creator_user_token: None,
+    creator_set_visibility,
+    public_bucket_directory_hash: public_upload_path.get_object_hash(),
+    maybe_public_bucket_prefix,
+    maybe_public_bucket_extension,
+  }).await?;
+
+  info!("New media file token: {:?}", media_file_token);
+
+  let bucket_url = format!(
+    "https://storage.cloud.google.com/{}{}",
+    public_bucket_name,
+    public_upload_path.get_full_object_path_str()
+  );
+
+  let gcp_url = format!(
+    "https://console.cloud.google.com/storage/browser/_details/{}{}",
+    public_bucket_name,
+    public_upload_path.get_full_object_path_str()
+  );
+
+  println!("\n\n   ========== FILE INFO ==========\n\n");
+  println!(" token: {}", media_file_token);
+  println!(" URL: {}", bucket_url);
+  println!(" GCP Admin: {}", gcp_url);
+  println!("\n");
+
+  Ok(())
+}

crates/lib/easyenv/src/init.rs

@@ -11,6 +11,16 @@ pub fn init_all_with_default_logging(default_if_absent: Option<&str>) {
   init_env_logger(default_if_absent)
 }
 
+/// Read env configs from a filename.
+pub fn from_filename<P: AsRef<Path>>(filename: P) -> Result<(), InitError> {
+  let _path = dotenv::from_filename(&filename)
+      .map_err(|err| {
+        error!("Could not read env config from path {:?} : {:?}", filename.as_ref(), err);
+        InitError::DotEnvError
+      })?;
+  Ok(())
+}
+
 /// Try loading in an environment file across many search paths (first found wins).
 /// Returns an error if no file could be read or if in attempting to read a file there was an error.
 pub fn read_env_config_from_filename_and_paths<P: AsRef<Path>, Q: AsRef<Path>>(filename: P, paths: &[Q]) -> Result<(), InitError> {

crates/lib/easyenv/src/lib.rs

@@ -32,6 +32,7 @@
 // It's easy to just keep them exported here to maintain compatibility.
 pub use error::EnvError;
 pub use error::InitError;
+pub use init::from_filename;
 pub use init::init_all_with_default_logging;
 pub use init::maybe_read_env_config_from_filename_and_paths;
 pub use init::read_env_config_from_filename_and_paths;

crates/lib/files/filesys/src/file_read_bytes.rs

@@ -0,0 +1,16 @@
+use std::fs::File;
+use std::io;
+use std::io::{BufReader, Read};
+use std::path::Path;
+
+/// Read a file into a byte buffer
+pub fn file_read_bytes<P: AsRef<Path>>(file_path: P) -> io::Result<Vec<u8>> {
+  let f = File::open(file_path)?;
+  let mut reader = BufReader::new(f);
+  let mut buffer = Vec::new();
+
+  // Read file into vector.
+  reader.read_to_end(&mut buffer)?;
+
+  Ok(buffer)
+}

crates/lib/files/filesys/src/lib.rs

@@ -21,6 +21,7 @@
 pub mod create_dir_all_if_missing;
 pub mod directory_exists;
 pub mod file_exists;
+pub mod file_read_bytes;
 pub mod file_size;
 pub mod filename_concat;
 pub mod path_to_string;

crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs

@@ -0,0 +1,111 @@
+use anyhow::anyhow;
+use log::error;
+use sqlx;
+use sqlx::MySqlPool;
+
+use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
+use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
+use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
+use enums::by_table::media_files::media_file_type::MediaFileType;
+use enums::common::visibility::Visibility;
+use errors::AnyhowResult;
+use tokens::tokens::media_files::MediaFileToken;
+use tokens::tokens::users::UserToken;
+
+pub struct InsertArgs<'a> {
+  pub pool: &'a MySqlPool,
+
+  pub media_file_type: MediaFileType,
+  pub maybe_mime_type: Option<&'a str>,
+  pub file_size_bytes: u64,
+  pub sha256_checksum: &'a str,
+
+  pub maybe_creator_user_token: Option<UserToken>,
+  pub creator_set_visibility: Visibility,
+
+  pub public_bucket_directory_hash: &'a str,
+  pub maybe_public_bucket_prefix: Option<&'a str>,
+  pub maybe_public_bucket_extension: Option<&'a str>,
+}
+
+// NB: Do not use from website / jobs. This is meant to be used from the CLI tool,
+// and shoudl allow for a more free-form interface.
+pub async fn insert_media_file_from_cli_tool(
+  args: InsertArgs<'_>
+) -> AnyhowResult<(MediaFileToken, u64)>
+{
+  let media_file_token = MediaFileToken::generate();
+
+  let mut transaction = args.pool.begin().await?;
+
+  const ORIGIN_CATEGORY : MediaFileOriginCategory = MediaFileOriginCategory::Upload;
+  const ORIGIN_PRODUCT_CATEGORY : MediaFileOriginProductCategory = MediaFileOriginProductCategory::Unknown;
+
+  // TODO(bt,2023-10-19): UserToken should automatically serialize to the DB. What's going on?
+  let maybe_user_token = args.maybe_creator_user_token.map(|u| u.to_string());
+
+  let record_id = {
+    let query_result = sqlx::query!(
+        r#"
+INSERT INTO media_files
+SET
+  token = ?,
+
+  origin_category = ?,
+  origin_product_category = ?,
+
+  media_type = ?,
+  maybe_mime_type = ?,
+  file_size_bytes = ?,
+  checksum_sha2 = ?,
+
+  public_bucket_directory_hash = ?,
+  maybe_public_bucket_prefix = ?,
+  maybe_public_bucket_extension = ?,
+
+  maybe_creator_user_token = ?,
+  creator_ip_address = ?,
+
+  creator_set_visibility = ?
+        "#,
+      media_file_token,
+
+      ORIGIN_CATEGORY,
+      ORIGIN_PRODUCT_CATEGORY,
+
+      args.media_file_type,
+      args.maybe_mime_type,
+      args.file_size_bytes,
+      args.sha256_checksum,
+
+      args.public_bucket_directory_hash,
+      args.maybe_public_bucket_prefix,
+      args.maybe_public_bucket_extension,
+
+      maybe_user_token,
+      "127.0.0.1",
+
+      args.creator_set_visibility.to_str(),
+    )
+        .execute(&mut transaction)
+        .await;
+
+    let record_id = match query_result {
+      Ok(res) => {
+        res.last_insert_id()
+      },
+      Err(err) => {
+        // TODO: handle better
+        //transaction.rollback().await?;
+        error!("Error executing query: {:?}", err);
+        return Err(anyhow!("Mysql error: {:?}", err));
+      }
+    };
+
+    record_id
+  };
+
+  transaction.commit().await?;
+
+  Ok((media_file_token, record_id))
+}

crates/lib/mysql_queries/src/queries/media_files/mod.rs

@@ -1,2 +1,3 @@
 pub mod get_media_file;
-pub mod insert_media_file_from_face_animation;
+pub mod insert_media_file_from_face_animation;
+pub mod insert_media_file_from_cli_tool;