upload for analysis

Cargo.lock

@@ -1730,6 +1730,14 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "bucket_client"
+version = "0.0.1"
+dependencies = [
+ "rust-s3",
+ "workspace-build-acceleration",
+]
+
 [[package]]
 name = "bucket_paths"
 version = "0.0.1"
@@ -11581,6 +11589,7 @@ dependencies = [
  "billing_artcraft_component",
  "billing_component",
  "bootstrap",
+ "bucket_client",
  "bucket_paths",
  "bytes",
  "chrono",

Cargo.toml

@@ -10,6 +10,7 @@ members = [
   "crates/api_clients/artcraft/artcraft_api_keys",
   "crates/api_clients/artcraft/artcraft_client",
   "crates/api_clients/artcraft/artcraft_router",
+  "crates/api_clients/bucket_client",
   "crates/api_clients/cloud_storage",
   "crates/api_clients/comet_api_client",
   "crates/api_clients/datadog_client",
@@ -239,6 +240,7 @@ artcraft_api_keys = { path = "crates/api_clients/artcraft/artcraft_api_keys" }
 artcraft_client = { path = "crates/api_clients/artcraft/artcraft_client" }
 artcraft_router = { path = "crates/api_clients/artcraft/artcraft_router" }
 bootstrap = { path = "crates/lib/bootstrap" }
+bucket_client = { path = "crates/api_clients/bucket_client" }
 bucket_paths = { path = "crates/schema/buckets/bucket_paths" }
 build_metadata = { path = "crates/lib/build_metadata" }
 cloud_storage = { path = "crates/api_clients/cloud_storage" }

crates/api_clients/bucket_client/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "bucket_client"
+edition = "2024"
+version = "0.0.1"
+publish = false
+
+[lib]
+name = "bucket_client"
+path = "src/lib.rs"
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["workspace-build-acceleration"]
+
+[dependencies]
+# cargo-hakari workspace hack (faster compile times)
+workspace-build-acceleration.workspace = true
+
+# External
+rust-s3.workspace = true

crates/api_clients/bucket_client/src/bucket_client.rs

@@ -0,0 +1,66 @@
+use s3::Bucket;
+
+use crate::bucket_client_error::BucketClientError;
+use crate::object_name::ObjectName;
+
+/// Arguments for [`BucketClient::upload_file_bytes`].
+pub struct UploadFileBytesArgs<'a, S: AsRef<str>> {
+  /// The destination object key (normalized via [`ObjectName`]).
+  pub object_name: S,
+  /// The raw bytes to upload.
+  pub bytes: &'a [u8],
+  /// Optional MIME type. Defaults to `application/octet-stream` when `None`.
+  pub content_type: Option<&'a str>,
+}
+
+/// A client for a single S3-compatible bucket. Cheap to clone. Build one with
+/// [`crate::BucketClientBuilder`].
+#[derive(Clone)]
+pub struct BucketClient {
+  bucket: Bucket,
+}
+
+impl BucketClient {
+  /// Wrap a fully-configured `Bucket`. Internal — use
+  /// [`crate::BucketClientBuilder`] to construct a client.
+  pub(crate) fn from_bucket(bucket: Bucket) -> Self {
+    Self { bucket }
+  }
+
+  /// The name of the bucket this client targets.
+  pub fn bucket_name(&self) -> &str {
+    self.bucket.name.as_str()
+  }
+
+  /// Upload raw bytes to `object_name`, optionally with an explicit content type.
+  ///
+  /// Returns [`BucketClientError::UploadFailed`] if the store responds with a
+  /// non-2xx status, or [`BucketClientError::S3Error`] on a transport/protocol
+  /// error.
+  pub async fn upload_file_bytes<S: AsRef<str>>(
+    &self,
+    args: UploadFileBytesArgs<'_, S>,
+  ) -> Result<(), BucketClientError> {
+    let object_name = ObjectName::new(&args.object_name);
+
+    let response = match args.content_type {
+      Some(content_type) => {
+        self
+          .bucket
+          .put_object_with_content_type(object_name.as_str(), args.bytes, content_type)
+          .await?
+      }
+      None => self.bucket.put_object(object_name.as_str(), args.bytes).await?,
+    };
+
+    let status_code = response.status_code();
+    if !(200..300).contains(&status_code) {
+      return Err(BucketClientError::UploadFailed {
+        status_code,
+        message: String::from_utf8_lossy(response.bytes()).into_owned(),
+      });
+    }
+
+    Ok(())
+  }
+}

crates/api_clients/bucket_client/src/bucket_client_builder.rs

@@ -0,0 +1,119 @@
+use std::time::Duration;
+
+use s3::creds::Credentials;
+use s3::region::Region;
+use s3::Bucket;
+
+use crate::bucket_client::BucketClient;
+use crate::bucket_client_error::BucketClientError;
+
+/// Builder for [`BucketClient`].
+///
+/// `access_key`, `secret_key`, `region_name`, and `bucket_name` are required;
+/// `endpoint` and `bucket_request_timeout` are optional.
+///
+/// - With an `endpoint` set, the bucket targets a custom S3-compatible host
+///   (R2, GCS, MinIO, etc.) via `Region::Custom`, and uses path-style addressing
+///   (subdomain-style for Google Cloud Storage).
+/// - Without an `endpoint`, the `region_name` is parsed as an AWS region
+///   (e.g. `"us-east-1"`), which determines the AWS endpoint automatically.
+#[derive(Debug, Default, Clone)]
+pub struct BucketClientBuilder {
+  access_key: Option<String>,
+  secret_key: Option<String>,
+  region_name: Option<String>,
+  bucket_name: Option<String>,
+  endpoint: Option<String>,
+  bucket_request_timeout: Option<Duration>,
+}
+
+impl BucketClientBuilder {
+  pub fn new() -> Self {
+    Self::default()
+  }
+
+  pub fn access_key(mut self, access_key: impl Into<String>) -> Self {
+    self.access_key = Some(access_key.into());
+    self
+  }
+
+  pub fn secret_key(mut self, secret_key: impl Into<String>) -> Self {
+    self.secret_key = Some(secret_key.into());
+    self
+  }
+
+  pub fn region_name(mut self, region_name: impl Into<String>) -> Self {
+    self.region_name = Some(region_name.into());
+    self
+  }
+
+  pub fn bucket_name(mut self, bucket_name: impl Into<String>) -> Self {
+    self.bucket_name = Some(bucket_name.into());
+    self
+  }
+
+  /// Optional custom S3-compatible endpoint (e.g. an R2/GCS/MinIO URL). When
+  /// unset, an AWS endpoint is derived from `region_name`.
+  pub fn endpoint(mut self, endpoint: impl Into<String>) -> Self {
+    self.endpoint = Some(endpoint.into());
+    self
+  }
+
+  pub fn bucket_request_timeout(mut self, timeout: Duration) -> Self {
+    self.bucket_request_timeout = Some(timeout);
+    self
+  }
+
+  /// Validate the configuration and construct a [`BucketClient`].
+  pub fn build(self) -> Result<BucketClient, BucketClientError> {
+    let access_key = require(self.access_key, "access_key")?;
+    let secret_key = require(self.secret_key, "secret_key")?;
+    let region_name = require(self.region_name, "region_name")?;
+    let bucket_name = require(self.bucket_name, "bucket_name")?;
+
+    let credentials = Credentials {
+      access_key: Some(access_key),
+      secret_key: Some(secret_key),
+      security_token: None,
+      session_token: None,
+      expiration: None,
+    };
+
+    // A custom endpoint targets an S3-compatible host via `Region::Custom`;
+    // otherwise the region name is parsed as a standard AWS region.
+    let region = match self.endpoint.as_deref() {
+      Some(endpoint) => Region::Custom {
+        region: region_name.clone(),
+        endpoint: endpoint.to_string(),
+      },
+      None => region_name.parse().map_err(|err| {
+        BucketClientError::ClientBuilderSetupError(format!(
+          "invalid region {region_name:?}: {err:?}"
+        ))
+      })?,
+    };
+
+    let mut bucket = Bucket::new(&bucket_name, region, credentials)?;
+    bucket.set_request_timeout(self.bucket_request_timeout);
+
+    // Addressing style only matters for custom endpoints: Google Cloud Storage
+    // needs subdomain (virtual-hosted) style; other S3-compatible stores use
+    // path style. AWS uses the crate default (virtual-hosted).
+    if let Some(endpoint) = self.endpoint.as_deref() {
+      if endpoint == "https://storage.googleapis.com" {
+        bucket.set_subdomain_style();
+      } else {
+        bucket.set_path_style();
+      }
+    }
+
+    Ok(BucketClient::from_bucket(*bucket))
+  }
+}
+
+/// Resolve a required builder field, or report which one was missing.
+fn require(value: Option<String>, field: &str) -> Result<String, BucketClientError> {
+  value.ok_or_else(|| {
+    BucketClientError::ClientBuilderSetupError(format!("{field} is required"))
+  })
+}

crates/api_clients/bucket_client/src/bucket_client_error.rs

@@ -0,0 +1,56 @@
+use std::fmt;
+use std::io;
+
+/// Errors produced by [`crate::BucketClient`] and [`crate::BucketClientBuilder`].
+#[derive(Debug)]
+pub enum BucketClientError {
+  /// An error surfaced by the underlying `rust-s3` client.
+  S3Error(s3::error::S3Error),
+
+  /// A local I/O error (e.g. while reading bytes to upload).
+  IoError(io::Error),
+
+  /// The builder was misconfigured — a required field was missing, or client
+  /// construction failed for a reason we can describe.
+  ClientBuilderSetupError(String),
+
+  /// The object store accepted the request but returned a non-success status.
+  UploadFailed { status_code: u16, message: String },
+}
+
+impl fmt::Display for BucketClientError {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Self::S3Error(err) => write!(f, "S3 error: {}", err),
+      Self::IoError(err) => write!(f, "I/O error: {}", err),
+      Self::ClientBuilderSetupError(message) => {
+        write!(f, "bucket client setup error: {}", message)
+      }
+      Self::UploadFailed { status_code, message } => {
+        write!(f, "upload failed (HTTP {}): {}", status_code, message)
+      }
+    }
+  }
+}
+
+impl std::error::Error for BucketClientError {
+  fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+    match self {
+      Self::S3Error(err) => Some(err),
+      Self::IoError(err) => Some(err),
+      _ => None,
+    }
+  }
+}
+
+impl From<s3::error::S3Error> for BucketClientError {
+  fn from(err: s3::error::S3Error) -> Self {
+    Self::S3Error(err)
+  }
+}
+
+impl From<io::Error> for BucketClientError {
+  fn from(err: io::Error) -> Self {
+    Self::IoError(err)
+  }
+}

crates/api_clients/bucket_client/src/lib.rs

@@ -0,0 +1,15 @@
+//! bucket_client
+//!
+//! A small, ergonomic client for S3-compatible object storage, built on the
+//! `rust-s3` crate. Construct one with [`BucketClientBuilder`] and upload bytes
+//! with [`BucketClient::upload_file_bytes`].
+
+pub mod bucket_client;
+pub mod bucket_client_builder;
+pub mod bucket_client_error;
+pub mod object_name;
+
+pub use bucket_client::{BucketClient, UploadFileBytesArgs};
+pub use bucket_client_builder::BucketClientBuilder;
+pub use bucket_client_error::BucketClientError;
+pub use object_name::ObjectName;

crates/api_clients/bucket_client/src/object_name.rs

@@ -0,0 +1,52 @@
+use std::fmt;
+
+/// A normalized object key for the bucket.
+///
+/// Leading slashes are stripped: S3/R2 treat a leading `/` as an empty leading
+/// path segment, which silently produces surprising keys (e.g. `/foo.mp4`
+/// becomes an object literally named `/foo.mp4`). Normalizing here keeps stored
+/// keys consistent regardless of how callers format the name.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ObjectName(String);
+
+impl ObjectName {
+  /// Build a normalized object name from any string-like value.
+  pub fn new(raw: impl AsRef<str>) -> Self {
+    ObjectName(raw.as_ref().trim_start_matches('/').to_string())
+  }
+
+  /// The normalized key as a string slice.
+  pub fn as_str(&self) -> &str {
+    &self.0
+  }
+}
+
+impl fmt::Display for ObjectName {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{}", self.0)
+  }
+}
+
+impl AsRef<str> for ObjectName {
+  fn as_ref(&self) -> &str {
+    &self.0
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn strips_leading_slashes() {
+    assert_eq!(ObjectName::new("/foo/bar.mp4").as_str(), "foo/bar.mp4");
+    assert_eq!(ObjectName::new("///a").as_str(), "a");
+    assert_eq!(ObjectName::new("already/clean.txt").as_str(), "already/clean.txt");
+  }
+
+  #[test]
+  fn accepts_string_and_str() {
+    assert_eq!(ObjectName::new(String::from("x.bin")).as_str(), "x.bin");
+    assert_eq!(ObjectName::new("x.bin").as_str(), "x.bin");
+  }
+}

crates/lib/sha1_hash/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sha1_hash"
-edition = "2021"
+edition = "2024"
 version = "0.0.1"
 publish = false
 

crates/service/web/storyteller_web/Cargo.toml

@@ -41,6 +41,7 @@ artcraft_router.workspace = true
 sha1_hash.workspace = true
 video_info.workspace = true
 bootstrap.workspace = true
+bucket_client.workspace = true
 bucket_paths.workspace = true
 cloud_storage.workspace = true
 collections.workspace = true

crates/service/web/storyteller_web/src/http_server/endpoints/video_info/video_info_upload_handler.rs

@@ -17,6 +17,7 @@ use log::{error, warn};
 use utoipa::ToSchema;
 
 use artcraft_api_defs::video_info::upload::VideoInfoUploadResponse;
+use bucket_client::UploadFileBytesArgs;
 use ffmpeg_utils::ffprobe::ffprobe_get_dimensions::ffprobe_get_dimensions;
 use http_server_common::request::get_request_ip::get_request_ip;
 use mysql_queries::queries::uploaded_videos::get_uploaded_video_by_sha1_checksum::{
@@ -105,6 +106,23 @@ pub async fn video_info_upload_handler(
 
   let provenance = to_provenance(&parse_result, maybe_encoder);
 
+  // Archive the original bytes to the Seedance video bucket, if configured.
+  // Done before moving `bytes` into the ffprobe task, and before acquiring a DB
+  // connection. Fail-soft: a storage error must not fail provenance detection.
+  if let Some(bucket) = server_state.seedance_video_bucket.as_ref() {
+    let object_name = seedance_video_object_name(&sha1_checksum);
+    if let Err(err) = bucket
+      .upload_file_bytes(UploadFileBytesArgs {
+        object_name: object_name.as_str(),
+        bytes: &bytes,
+        content_type: Some("video/mp4"),
+      })
+      .await
+    {
+      warn!("seedance video bucket upload failed for {}: {:?}", object_name, err);
+    }
+  }
+
   // ffprobe is a blocking subprocess — run it off the async executor, and BEFORE
   // acquiring a DB connection (never hold a pooled connection across this).
   //
@@ -180,6 +198,19 @@ pub async fn video_info_upload_handler(
   }))
 }
 
+/// Object key for archiving an uploaded video, sharded into four directory
+/// levels by the first four characters of the checksum:
+/// `uploads/{c0}/{c1}/{c2}/{c3}/{checksum}.mp4`
+/// (e.g. checksum `f0a2cda0…` → `uploads/f/0/a/2/f0a2cda0….mp4`).
+fn seedance_video_object_name(sha1_checksum: &str) -> String {
+  let shard: String = sha1_checksum
+    .chars()
+    .take(4)
+    .map(|character| format!("{character}/"))
+    .collect();
+  format!("uploads/{shard}{sha1_checksum}.mp4")
+}
+
 /// Write the bytes to a temp file and ffprobe its video dimensions. Fail-soft:
 /// returns `None` on any failure (not a probeable video, ffprobe missing, I/O
 /// error) so the caller can proceed without dimensions. Never panics.
@@ -208,3 +239,16 @@ fn probe_dimensions(bytes: &[u8]) -> Option<(u32, u32)> {
     }
   }
 }
+
+#[cfg(test)]
+mod tests {
+  use super::seedance_video_object_name;
+
+  #[test]
+  fn shards_object_name_by_first_four_checksum_chars() {
+    assert_eq!(
+      seedance_video_object_name("f0a2cda0deadbeef"),
+      "uploads/f/0/a/2/f0a2cda0deadbeef.mp4"
+    );
+  }
+}

crates/service/web/storyteller_web/src/startup/build_dependencies.rs

@@ -34,6 +34,7 @@ use crate::startup::setup_bans::{
   load_static_container_ip_bans, load_troll_user_token_bans,
 };
 use crate::startup::setup_inference_providers::setup_inference_providers;
+use crate::startup::setup_seedance_video_bucket::setup_seedance_video_bucket;
 use crate::startup::setup_static_feature_flags::setup_static_feature_flags;
 use crate::startup::setup_stripe_artcraft::setup_stripe_artcraft;
 use crate::startup::setup_stripe_fakeyou::setup_stripe_fakeyou;
@@ -236,6 +237,7 @@ pub async fn setup_dependencies(server_hostname: &str) -> AnyhowResult<SetupResu
     private_bucket_client,
     public_bucket_client,
     auto_gc_bucket_client,
+    seedance_video_bucket: setup_seedance_video_bucket(),
     audio_uploads_bucket_root,
     sort_key_crypto,
     opaque_cursors: opaque_cursor_encoder,

crates/service/web/storyteller_web/src/startup/mod.rs

@@ -2,6 +2,7 @@ pub mod build_dependencies;
 pub mod setup_bans;
 pub mod setup_metrics;
 pub mod setup_pager;
+pub mod setup_seedance_video_bucket;
 pub mod setup_disabled_endpoints;
 pub mod setup_inference_providers;
 pub mod setup_static_feature_flags;

crates/service/web/storyteller_web/src/startup/setup_seedance_video_bucket.rs

@@ -0,0 +1,62 @@
+//! Optional setup for the Seedance video archive bucket.
+//!
+//! Reads the five `SEEDANCE_VIDEO_BUCKET_*` env vars. All must be present to
+//! build a client; otherwise the client is `None` and archival uploads are
+//! simply skipped. A *partial* configuration is treated as a misconfiguration
+//! (warned, not built).
+
+use std::env;
+
+use bucket_client::{BucketClient, BucketClientBuilder};
+use log::{info, warn};
+
+/// Build the optional Seedance video bucket client from the environment.
+pub fn setup_seedance_video_bucket() -> Option<BucketClient> {
+  let bucket_name = read_non_empty_env("SEEDANCE_VIDEO_BUCKET_NAME");
+  let region = read_non_empty_env("SEEDANCE_VIDEO_BUCKET_REGION");
+  let endpoint = read_non_empty_env("SEEDANCE_VIDEO_BUCKET_ENDPOINT");
+  let access_key = read_non_empty_env("SEEDANCE_VIDEO_BUCKET_ACCESS_KEY_ID");
+  let secret_key = read_non_empty_env("SEEDANCE_VIDEO_BUCKET_SECRET_ACCESS_KEY");
+
+  match (bucket_name, region, endpoint, access_key, secret_key) {
+    (Some(bucket_name), Some(region), Some(endpoint), Some(access_key), Some(secret_key)) => {
+      match BucketClientBuilder::new()
+        .access_key(access_key)
+        .secret_key(secret_key)
+        .region_name(region)
+        .bucket_name(&bucket_name)
+        .endpoint(endpoint)
+        .build()
+      {
+        Ok(client) => {
+          info!("Seedance video bucket configured: {}", bucket_name);
+          Some(client)
+        }
+        Err(err) => {
+          warn!("Seedance video bucket env present but client setup failed: {}", err);
+          None
+        }
+      }
+    }
+    (None, None, None, None, None) => {
+      info!("Seedance video bucket not configured; archival uploads disabled.");
+      None
+    }
+    _ => {
+      warn!(
+        "Seedance video bucket is only partially configured \
+         (need all of SEEDANCE_VIDEO_BUCKET_NAME / _REGION / _ENDPOINT / \
+         _ACCESS_KEY_ID / _SECRET_ACCESS_KEY); skipping setup."
+      );
+      None
+    }
+  }
+}
+
+/// Read an env var, treating unset or whitespace-only as absent.
+fn read_non_empty_env(name: &str) -> Option<String> {
+  match env::var(name) {
+    Ok(value) if !value.trim().is_empty() => Some(value),
+    _ => None,
+  }
+}

crates/service/web/storyteller_web/src/state/server_state.rs

@@ -25,6 +25,7 @@ use billing_artcraft_component::utils::artcraft_stripe_config::ArtcraftStripeCon
 use billing_component::stripe::stripe_config::StripeConfig;
 use chrono::{DateTime, Utc};
 use cloud_storage::bucket_client::BucketClient;
+use bucket_client::BucketClient as SeedanceVideoBucketClient;
 use elasticsearch::Elasticsearch;
 use memory_caching::arc_ttl_sieve::ArcTtlSieve;
 use memory_caching::single_item_ttl_cache::SingleItemTtlCache;
@@ -88,6 +89,10 @@ pub struct ServerState {
   pub public_bucket_client: BucketClient,
   pub auto_gc_bucket_client: BucketClient,
 
+  /// Optional archive bucket for uploaded Seedance videos. `None` when the
+  /// `SEEDANCE_VIDEO_BUCKET_*` env vars aren't configured.
+  pub seedance_video_bucket: Option<SeedanceVideoBucketClient>,
+
   pub inference_providers: InferenceProviders,
 
   pub resend: ResendData,