mirror of
https://github.com/storytold/artcraft.git
synced 2026-10-20 17:02:53 -04:00
thumbnail job (#1157)
This commit is contained in:
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"query": "\n UPDATE media_files\n SET maybe_thumbnail_version = ?\n WHERE token = ?\n LIMIT 1\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Right": 2
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "87d7233bb9a4bcae7c9b7d55bb93fab182343847f0bcf8d8ce849f3a2dcfb704"
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"query": "\nSELECT\n id,\n token as `token: MediaFileToken`,\n created_at as `created_at: DateTime<Utc>`,\n maybe_thumbnail_version as `maybe_thumbnail_version: u8`,\n public_bucket_directory_hash,\n maybe_public_bucket_prefix,\n maybe_public_bucket_extension\nFROM media_files\nWHERE\n id >= (SELECT COALESCE(MIN(id), 0) FROM media_files WHERE created_at >= NOW() - INTERVAL ? HOUR)\n AND id < ?\n AND media_class = ?\n AND maybe_thumbnail_version IS NULL\n AND user_deleted_at IS NULL\n AND mod_deleted_at IS NULL\nORDER BY id DESC\nLIMIT ?\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "id",
|
||||
"type_info": {
|
||||
"type": "LongLong",
|
||||
"flags": "NOT_NULL | PRIMARY_KEY | AUTO_INCREMENT",
|
||||
"char_set": 63,
|
||||
"max_size": 20
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "token: MediaFileToken",
|
||||
"type_info": {
|
||||
"type": "VarString",
|
||||
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
|
||||
"char_set": 224,
|
||||
"max_size": 128
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "created_at: DateTime<Utc>",
|
||||
"type_info": {
|
||||
"type": "Timestamp",
|
||||
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP",
|
||||
"char_set": 63,
|
||||
"max_size": 19
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "maybe_thumbnail_version: u8",
|
||||
"type_info": {
|
||||
"type": "Tiny",
|
||||
"flags": "UNSIGNED",
|
||||
"char_set": 63,
|
||||
"max_size": 3
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "public_bucket_directory_hash",
|
||||
"type_info": {
|
||||
"type": "VarString",
|
||||
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
|
||||
"char_set": 224,
|
||||
"max_size": 128
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "maybe_public_bucket_prefix",
|
||||
"type_info": {
|
||||
"type": "VarString",
|
||||
"flags": "BINARY",
|
||||
"char_set": 224,
|
||||
"max_size": 64
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "maybe_public_bucket_extension",
|
||||
"type_info": {
|
||||
"type": "VarString",
|
||||
"flags": "BINARY",
|
||||
"char_set": 224,
|
||||
"max_size": 64
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Right": 4
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "daed2dcd03b3c9fbdb9057b743df078f43e2af87a2c31f418234f98da5ff979f"
|
||||
}
|
||||
Generated
+43
-1
@@ -4242,6 +4242,19 @@ dependencies = [
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ffmpeg_utils"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"errors",
|
||||
"ffprobe",
|
||||
"log",
|
||||
"tempdir",
|
||||
"testing",
|
||||
"workspace-build-acceleration",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ffprobe"
|
||||
version = "0.4.0"
|
||||
@@ -5947,6 +5960,7 @@ dependencies = [
|
||||
"easyenv",
|
||||
"enums",
|
||||
"errors",
|
||||
"ffmpeg_utils",
|
||||
"filesys",
|
||||
"google_drive_common",
|
||||
"hashing",
|
||||
@@ -11567,6 +11581,7 @@ dependencies = [
|
||||
"enums_public",
|
||||
"errors",
|
||||
"fal_client",
|
||||
"ffmpeg_utils",
|
||||
"filesys",
|
||||
"futures 0.1.31",
|
||||
"futures 0.3.31",
|
||||
@@ -13686,12 +13701,39 @@ version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "video-thumbnail-job"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"actix-web",
|
||||
"actix_helpers",
|
||||
"anyhow",
|
||||
"bootstrap",
|
||||
"bucket_paths",
|
||||
"cloud_storage",
|
||||
"concurrency",
|
||||
"config",
|
||||
"easyenv",
|
||||
"errors",
|
||||
"ffmpeg_utils",
|
||||
"jobs_common",
|
||||
"log",
|
||||
"mysql_queries",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"server_environment",
|
||||
"sqlx",
|
||||
"tempdir",
|
||||
"tokio",
|
||||
"workspace-build-acceleration",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "videos"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"errors",
|
||||
"ffprobe",
|
||||
"log",
|
||||
"mp4",
|
||||
"testing",
|
||||
|
||||
+13
-5
@@ -44,7 +44,7 @@ members = [
|
||||
"crates/lib/actix_cors_configs",
|
||||
"crates/lib/actix_helpers",
|
||||
"crates/lib/bootstrap",
|
||||
"crates/lib/bucket_paths",
|
||||
"crates/schema/buckets/bucket_paths",
|
||||
"crates/lib/build_metadata",
|
||||
"crates/lib/caching/memory_caching",
|
||||
"crates/lib/caching/memory_store",
|
||||
@@ -65,6 +65,7 @@ members = [
|
||||
"crates/lib/easyenv", # NB: Vendored version of https://github.com/echelon/easyenv.rs that we'll improve.
|
||||
"crates/lib/email_sender",
|
||||
"crates/lib/errors",
|
||||
"crates/lib/ffmpeg_utils",
|
||||
"crates/lib/files/archives",
|
||||
"crates/lib/files/file_to_base64_url",
|
||||
"crates/lib/files/filesys",
|
||||
@@ -122,6 +123,7 @@ members = [
|
||||
"crates/service/job/es-update-job",
|
||||
"crates/service/job/inference_job",
|
||||
"crates/service/job/seedance2_pro_job",
|
||||
"crates/service/job/video_thumbnail_job",
|
||||
"crates/service/job/worldlabs_job",
|
||||
"crates/service/job/tts_download_job",
|
||||
"crates/service/job/tts_inference_job",
|
||||
@@ -212,15 +214,19 @@ workspace-build-acceleration = { path = "crates/workspace-build-acceleration" }
|
||||
# Setting paths here ensures we don't need to do so in every crate's Cargo.toml
|
||||
# It makes refactoring much easier.
|
||||
actix_artcraft = { path = "crates/lib/actix_artcraft" }
|
||||
artcraft_router = { path = "crates/api_clients/artcraft/artcraft_router" }
|
||||
actix_helpers = { path = "crates/lib/actix_helpers" }
|
||||
artcraft_api_defs = { path = "crates/api_clients/artcraft/artcraft_api_defs" }
|
||||
artcraft_client = { path = "crates/api_clients/artcraft/artcraft_client" }
|
||||
artcraft_router = { path = "crates/api_clients/artcraft/artcraft_router" }
|
||||
bootstrap = { path = "crates/lib/bootstrap" }
|
||||
bucket_paths = { path = "crates/schema/buckets/bucket_paths" }
|
||||
build_metadata = { path = "crates/lib/build_metadata" }
|
||||
cloud_storage = { path = "crates/api_clients/cloud_storage" }
|
||||
cloudflare_errors = { path = "crates/lib/cloudflare_errors" }
|
||||
cloudflare_mitigation = { path = "crates/lib/cloudflare_mitigation" }
|
||||
collections = { path = "crates/lib/data/collections" }
|
||||
component_traits = { path = "crates/service/plugins/component_traits" }
|
||||
concurrency = { path = "crates/lib/concurrency" }
|
||||
cookie_store = { path = "crates/lib/cookie_store" }
|
||||
cookies = { path = "crates/lib/cookies" }
|
||||
crockford = { path = "crates/lib/data/crockford" }
|
||||
@@ -228,6 +234,7 @@ easyenv = { path = "crates/lib/easyenv" }
|
||||
enums = { path = "crates/schema/public/enums" }
|
||||
enums_public = { path = "crates/schema/public/enums_public" }
|
||||
errors = { path = "crates/lib/errors" }
|
||||
ffmpeg_utils = { path = "crates/lib/ffmpeg_utils" }
|
||||
fal_client = { path = "crates/api_clients/fal_client" }
|
||||
file_to_base64_url = { path = "crates/lib/files/file_to_base64_url" }
|
||||
filesys = { path = "crates/lib/files/filesys" }
|
||||
@@ -235,18 +242,19 @@ grok_client = { path = "crates/api_clients/grok_client" }
|
||||
http_headers = { path = "crates/lib/http_headers" }
|
||||
idempotency = { path = "crates/lib/idempotency" }
|
||||
images = { path = "crates/lib/files/images" }
|
||||
jobs_common = { path = "crates/lib/jobs_common" }
|
||||
jwt_light = { path = "crates/lib/jwt_light" }
|
||||
memory_caching = { path = "crates/lib/caching/memory_caching" }
|
||||
memory_store = { path = "crates/lib/caching/memory_store" }
|
||||
midjourney_client = { path = "crates/api_clients/midjourney_client" }
|
||||
mimetypes = { path = "crates/lib/files/mimetypes" }
|
||||
openai_sora_client = { path = "crates/api_clients/openai_sora_client" }
|
||||
muapi_client = { path = "crates/api_clients/muapi_client" }
|
||||
seedance2pro = { path = "crates/api_clients/seedance2pro" }
|
||||
mysql_queries = { path = "crates/schema/database/mysql_queries" }
|
||||
openai_sora_client = { path = "crates/api_clients/openai_sora_client" }
|
||||
primitives = { path = "crates/lib/data/primitives" }
|
||||
seedance2pro = { path = "crates/api_clients/seedance2pro" }
|
||||
shared_service_components = { path = "crates/lib/shared_service_components" }
|
||||
sqlite_tasks = { path = "crates/schema/database/sqlite_tasks" }
|
||||
artcraft_client = { path = "crates/api_clients/artcraft/artcraft_client" }
|
||||
subprocess_common = { path = "crates/lib/subprocess_common" }
|
||||
test_data = { path = "crates/lib/test_data" }
|
||||
testing = { path = "crates/lib/testing" }
|
||||
|
||||
@@ -148,7 +148,13 @@ RUN SQLX_OFFLINE=true \
|
||||
$HOME/.cargo/bin/cargo build \
|
||||
--release \
|
||||
--bin worldlabs-job
|
||||
|
||||
|
||||
RUN SQLX_OFFLINE=true \
|
||||
LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
|
||||
$HOME/.cargo/bin/cargo build \
|
||||
--release \
|
||||
--bin video-thumbnail-job
|
||||
|
||||
RUN SQLX_OFFLINE=true \
|
||||
LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
|
||||
$HOME/.cargo/bin/cargo build \
|
||||
@@ -210,6 +216,7 @@ RUN echo -n ${GIT_SHA} > GIT_SHA
|
||||
COPY --from=builder /tmp/target/release/storyteller-web /
|
||||
COPY --from=builder /tmp/target/release/seedance2-pro-job /
|
||||
COPY --from=builder /tmp/target/release/worldlabs-job /
|
||||
COPY --from=builder /tmp/target/release/video-thumbnail-job /
|
||||
COPY --from=builder /tmp/target/release/dummy-service /
|
||||
COPY --from=builder /tmp/target/release/analytics-job /
|
||||
COPY --from=builder /tmp/target/release/email-sender-job /
|
||||
|
||||
@@ -20,7 +20,7 @@ normal = ["workspace-build-acceleration"]
|
||||
workspace-build-acceleration.workspace = true
|
||||
|
||||
# Internal
|
||||
bucket_paths = { path = "../../lib/bucket_paths" }
|
||||
bucket_paths.workspace = true
|
||||
crockford.workspace = true
|
||||
easyenv = { path = "../../lib/easyenv" }
|
||||
errors.workspace = true
|
||||
|
||||
@@ -20,7 +20,7 @@ normal = ["workspace-build-acceleration"]
|
||||
workspace-build-acceleration = { workspace = true }
|
||||
|
||||
# Internal
|
||||
bucket_paths = { path = "../../../lib/bucket_paths" }
|
||||
bucket_paths.workspace = true
|
||||
cloud_storage.workspace = true
|
||||
config = { path = "../../../lib/deprecated/config" }
|
||||
easyenv = { path = "../../../lib/easyenv" }
|
||||
|
||||
@@ -20,7 +20,7 @@ normal = ["workspace-build-acceleration"]
|
||||
workspace-build-acceleration = { workspace = true }
|
||||
|
||||
# Internal
|
||||
bucket_paths = { path = "../../../lib/bucket_paths" }
|
||||
bucket_paths.workspace = true
|
||||
cloud_storage.workspace = true
|
||||
config = { path = "../../../lib/deprecated/config" }
|
||||
easyenv = { path = "../../../lib/easyenv" }
|
||||
|
||||
@@ -20,7 +20,7 @@ normal = ["workspace-build-acceleration"]
|
||||
workspace-build-acceleration = { workspace = true }
|
||||
|
||||
# Internal
|
||||
bucket_paths = { path = "../../../lib/bucket_paths" }
|
||||
bucket_paths.workspace = true
|
||||
easyenv = { path = "../../../lib/easyenv" }
|
||||
enums = { path = "../../../schema/public/enums" }
|
||||
errors = { path = "../../../lib/errors" }
|
||||
|
||||
@@ -20,7 +20,7 @@ normal = ["workspace-build-acceleration"]
|
||||
workspace-build-acceleration = { workspace = true }
|
||||
|
||||
# Internal
|
||||
bucket_paths = { path = "../../../lib/bucket_paths" }
|
||||
bucket_paths.workspace = true
|
||||
config = { path = "../../../lib/deprecated/config" }
|
||||
easyenv = { path = "../../../lib/easyenv" }
|
||||
elasticsearch_schema = { path = "../../../schema/database/elasticsearch_schema" }
|
||||
|
||||
@@ -20,7 +20,7 @@ normal = ["workspace-build-acceleration"]
|
||||
workspace-build-acceleration.workspace = true
|
||||
|
||||
# Internal
|
||||
bucket_paths = { path = "../../../lib/bucket_paths" }
|
||||
bucket_paths.workspace = true
|
||||
cloud_storage.workspace = true
|
||||
config = { path = "../../../lib/deprecated/config" }
|
||||
easyenv = { path = "../../../lib/easyenv" }
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
/// Represents a bucket path.
|
||||
pub trait BucketPath {
|
||||
/// Return the rooted path of the object as a string
|
||||
fn to_rooted_string(&self) -> String;
|
||||
|
||||
/// Return the rooted path of the object as a path
|
||||
fn to_rooted_path(&self) -> PathBuf;
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
mod bucket_path;
|
||||
@@ -0,0 +1,33 @@
|
||||
[package]
|
||||
name = "ffmpeg_utils"
|
||||
description = "Common tools and commands for ffmpeg and ffprobe."
|
||||
edition = "2024"
|
||||
version = "0.0.1"
|
||||
authors = [
|
||||
"Brandon Thomas <bt@brand.io>",
|
||||
"Brandon Thomas <echelon@gmail.com>",
|
||||
]
|
||||
publish = false
|
||||
|
||||
[lib]
|
||||
name = "ffmpeg_utils"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
|
||||
[dependencies]
|
||||
# cargo-hakari workspace hack (faster compile times)
|
||||
workspace-build-acceleration.workspace = true
|
||||
|
||||
# Internal
|
||||
errors.workspace = true
|
||||
|
||||
# External
|
||||
anyhow.workspace = true
|
||||
ffprobe = "0.4.0"
|
||||
log.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempdir.workspace = true
|
||||
testing.workspace = true
|
||||
+1
-1
@@ -6,7 +6,7 @@ use std::process::Command;
|
||||
use std::time::Duration;
|
||||
|
||||
use errors::AnyhowResult;
|
||||
use videos::ffmpeg_timestamp_from_duration::ffmpeg_timestamp_from_duration;
|
||||
use crate::ffmpeg::ffmpeg_timestamp_from_duration::ffmpeg_timestamp_from_duration;
|
||||
|
||||
pub struct Args<'a> {
|
||||
pub video_input_path: &'a Path,
|
||||
@@ -0,0 +1,82 @@
|
||||
use anyhow::bail;
|
||||
use log::{error, info};
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
use errors::AnyhowResult;
|
||||
|
||||
pub struct FfmpegVideoFirstFrameToJpgThumbnailArgs<I: AsRef<Path>, O: AsRef<Path>> {
|
||||
pub input_video_path: I,
|
||||
pub output_jpg_path: O,
|
||||
}
|
||||
|
||||
/// Extract a single JPG frame from a video file, scaled down to fit within
|
||||
/// 320x320 while preserving aspect ratio.
|
||||
///
|
||||
/// Uses `-sseof -1` to seek 1 second before the end of the file, which avoids
|
||||
/// potentially-black first frames.
|
||||
pub fn ffmpeg_video_first_frame_to_jpg_thumbnail<I: AsRef<Path>, O: AsRef<Path>>(
|
||||
args: FfmpegVideoFirstFrameToJpgThumbnailArgs<I, O>,
|
||||
) -> AnyhowResult<()> {
|
||||
let mut command = Command::new("ffmpeg");
|
||||
|
||||
command
|
||||
.arg("-nostdin")
|
||||
.arg("-y")
|
||||
.arg("-sseof").arg("-1")
|
||||
.arg("-i").arg(args.input_video_path.as_ref())
|
||||
.arg("-vf").arg("scale=320:320:force_original_aspect_ratio=decrease")
|
||||
.arg("-vframes").arg("1")
|
||||
.arg(args.output_jpg_path.as_ref());
|
||||
|
||||
info!("Calling ffmpeg (jpg thumbnail)...");
|
||||
|
||||
let output = command.output()?;
|
||||
|
||||
if !output.status.success() {
|
||||
error!("bad exit status: {}", output.status);
|
||||
|
||||
let _r = std::io::stdout().write_all(&output.stdout);
|
||||
let _r = std::io::stderr().write_all(&output.stderr);
|
||||
|
||||
bail!("ffmpeg jpg thumbnail failed: {:?}", output.status.to_string());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::ffprobe::ffprobe_get_dimensions::ffprobe_get_dimensions;
|
||||
use tempdir::TempDir;
|
||||
use testing::test_file_path::test_file_path;
|
||||
|
||||
#[test]
|
||||
fn test_extract_jpg_thumbnail_from_mp4() {
|
||||
let input_path = test_file_path("test_data/video/mp4/golden_sun_garoh.mp4")
|
||||
.expect("test video should exist");
|
||||
|
||||
let temp_dir = TempDir::new_in("/tmp", "ffmpeg_jpg_thumb_test")
|
||||
.expect("should create temp dir");
|
||||
|
||||
let output_path = temp_dir.path().join("thumbnail.jpg");
|
||||
|
||||
ffmpeg_video_first_frame_to_jpg_thumbnail(FfmpegVideoFirstFrameToJpgThumbnailArgs {
|
||||
input_video_path: &input_path,
|
||||
output_jpg_path: &output_path,
|
||||
}).expect("ffmpeg should succeed");
|
||||
|
||||
assert!(output_path.exists(), "output jpg should exist");
|
||||
|
||||
// The input video is 640x480. scale=320:320:force_original_aspect_ratio=decrease
|
||||
// should produce 320x240 (maintaining 4:3 aspect ratio).
|
||||
let dimensions = ffprobe_get_dimensions(&output_path)
|
||||
.expect("ffprobe should succeed")
|
||||
.expect("dimensions should not be empty");
|
||||
|
||||
assert_eq!(dimensions.width, 320);
|
||||
assert_eq!(dimensions.height, 240);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
use anyhow::bail;
|
||||
use log::{error, info};
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
use errors::AnyhowResult;
|
||||
|
||||
pub struct FfmpegVideoGifPreviewArgs<I: AsRef<Path>, O: AsRef<Path>> {
|
||||
pub input_video_path: I,
|
||||
pub output_gif_path: O,
|
||||
}
|
||||
|
||||
/// Extract a short animated GIF preview from a video file.
|
||||
///
|
||||
/// Takes the first 5 seconds, resampled to 10 fps, scaled to 360px wide
|
||||
/// (preserving aspect ratio), with an optimized color palette.
|
||||
pub fn ffmpeg_video_gif_preview<I: AsRef<Path>, O: AsRef<Path>>(
|
||||
args: FfmpegVideoGifPreviewArgs<I, O>,
|
||||
) -> AnyhowResult<()> {
|
||||
let mut command = Command::new("ffmpeg");
|
||||
|
||||
command
|
||||
.arg("-nostdin")
|
||||
.arg("-y")
|
||||
.arg("-ss").arg("0")
|
||||
.arg("-to").arg("5")
|
||||
.arg("-i").arg(args.input_video_path.as_ref())
|
||||
.arg("-filter_complex")
|
||||
.arg("fps=10,scale=360:-1[s]; [s]split[a][b]; [a]palettegen[palette]; [b][palette]paletteuse")
|
||||
.arg(args.output_gif_path.as_ref());
|
||||
|
||||
info!("Calling ffmpeg (gif preview)...");
|
||||
|
||||
let output = command.output()?;
|
||||
|
||||
if !output.status.success() {
|
||||
error!("bad exit status: {}", output.status);
|
||||
|
||||
let _r = std::io::stdout().write_all(&output.stdout);
|
||||
let _r = std::io::stderr().write_all(&output.stderr);
|
||||
|
||||
bail!("ffmpeg gif preview failed: {:?}", output.status.to_string());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempdir::TempDir;
|
||||
use testing::test_file_path::test_file_path;
|
||||
|
||||
#[test]
|
||||
fn test_extract_gif_preview_from_mp4() {
|
||||
let input_path = test_file_path("test_data/video/mp4/golden_sun_garoh.mp4")
|
||||
.expect("test video should exist");
|
||||
|
||||
let temp_dir = TempDir::new_in("/tmp", "ffmpeg_gif_preview_test")
|
||||
.expect("should create temp dir");
|
||||
|
||||
let output_path = temp_dir.path().join("preview.gif");
|
||||
|
||||
ffmpeg_video_gif_preview(FfmpegVideoGifPreviewArgs {
|
||||
input_video_path: &input_path,
|
||||
output_gif_path: &output_path,
|
||||
}).expect("ffmpeg should succeed");
|
||||
|
||||
assert!(output_path.exists(), "output gif should exist");
|
||||
|
||||
// The input video is 640x480. scale=360:-1 should produce 360x270
|
||||
// (maintaining 4:3 aspect ratio).
|
||||
// Use ffprobe to verify the GIF dimensions.
|
||||
let result = ffprobe::ffprobe(&output_path)
|
||||
.expect("ffprobe should succeed");
|
||||
|
||||
let video_stream = result.streams.iter()
|
||||
.find(|s| s.codec_type.as_deref() == Some("video"))
|
||||
.expect("should have a video stream");
|
||||
|
||||
assert_eq!(video_stream.width, Some(360));
|
||||
assert_eq!(video_stream.height, Some(270));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
pub mod ffmpeg_timestamp_from_duration;
|
||||
pub mod ffmpeg_trim_and_resample;
|
||||
pub mod ffmpeg_video_first_frame_to_jpg_thumbnail;
|
||||
pub mod ffmpeg_video_gif_preview;
|
||||
+1
-1
@@ -45,7 +45,7 @@ pub fn ffprobe_get_dimensions(
|
||||
pub mod tests {
|
||||
use testing::test_file_path::test_file_path;
|
||||
|
||||
use crate::ffprobe_get_dimensions::ffprobe_get_dimensions;
|
||||
use super::ffprobe_get_dimensions;
|
||||
|
||||
#[test]
|
||||
pub fn test_decode_mp4() {
|
||||
+1
-3
@@ -114,9 +114,7 @@ fn parse_fps(ffprobe_fps: &str) -> AnyhowResult<f32> {
|
||||
pub mod tests {
|
||||
use testing::test_file_path::test_file_path;
|
||||
|
||||
use crate::ffprobe_get_info::ffprobe_get_info;
|
||||
|
||||
use super::parse_seconds;
|
||||
use super::{ffprobe_get_info, parse_seconds};
|
||||
|
||||
#[test]
|
||||
pub fn test_decode_mp4() {
|
||||
@@ -0,0 +1,2 @@
|
||||
pub mod ffprobe_get_dimensions;
|
||||
pub mod ffprobe_get_info;
|
||||
@@ -0,0 +1,22 @@
|
||||
//! ffmpeg_utils
|
||||
//!
|
||||
//! Common tools and commands for ffmpeg and ffprobe.
|
||||
//!
|
||||
|
||||
// Never allow these
|
||||
#![forbid(private_bounds)]
|
||||
#![forbid(private_interfaces)]
|
||||
#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
|
||||
|
||||
// Okay to toggle
|
||||
#![forbid(unreachable_patterns)]
|
||||
#![forbid(unused_imports)]
|
||||
#![forbid(unused_mut)]
|
||||
#![forbid(unused_variables)]
|
||||
|
||||
// Always allow
|
||||
#![allow(dead_code)]
|
||||
#![allow(non_snake_case)]
|
||||
|
||||
pub mod ffmpeg;
|
||||
pub mod ffprobe;
|
||||
@@ -21,14 +21,14 @@ normal = ["workspace-build-acceleration"]
|
||||
workspace-build-acceleration = { workspace = true }
|
||||
|
||||
# Internal
|
||||
errors = { workspace = true }
|
||||
errors.workspace = true
|
||||
|
||||
# External
|
||||
ffprobe = "0.4.0"
|
||||
log = { workspace = true }
|
||||
log.workspace = true
|
||||
|
||||
# TODO(bt,2024-08-08): We might drop `mp4` crate because it crashes on reading
|
||||
# some videos with the message "avcc not found"
|
||||
mp4 = "0.14.0"
|
||||
|
||||
[dev-dependencies]
|
||||
testing = { path = "../../testing" }
|
||||
testing.workspace = true
|
||||
|
||||
@@ -18,7 +18,4 @@
|
||||
#![allow(dead_code)]
|
||||
#![allow(non_snake_case)]
|
||||
|
||||
pub mod ffmpeg_timestamp_from_duration;
|
||||
pub mod ffprobe_get_dimensions;
|
||||
pub mod ffprobe_get_info;
|
||||
pub mod get_mp4_info;
|
||||
|
||||
@@ -21,7 +21,7 @@ normal = ["workspace-build-acceleration"]
|
||||
workspace-build-acceleration.workspace = true
|
||||
|
||||
# Internal
|
||||
bucket_paths = { path = "../../lib/bucket_paths" }
|
||||
bucket_paths.workspace = true
|
||||
cloud_storage.workspace = true
|
||||
enums.workspace = true
|
||||
errors.workspace = true
|
||||
|
||||
@@ -16,13 +16,15 @@ path = "src/lib.rs"
|
||||
normal = ["workspace-build-acceleration"]
|
||||
|
||||
[dependencies]
|
||||
# Workspace hack (faster compile times)
|
||||
workspace-build-acceleration = { workspace = true }
|
||||
# cargo-hakari workspace hack (faster compile times)
|
||||
workspace-build-acceleration.workspace = true
|
||||
|
||||
# Internal
|
||||
crockford = { path = "../data/crockford" }
|
||||
# Internal
|
||||
# (try not to add new deps as this is used by many crates)
|
||||
crockford.workspace = true
|
||||
|
||||
# External
|
||||
# (try not to add new deps as this is used by many crates)
|
||||
# (None yet)
|
||||
|
||||
[dev-dependencies]
|
||||
@@ -24,6 +24,5 @@
|
||||
#![allow(non_snake_case)]
|
||||
|
||||
pub mod legacy;
|
||||
pub mod pathing;
|
||||
pub mod traits;
|
||||
pub mod path_conventions;
|
||||
pub mod util;
|
||||
@@ -0,0 +1 @@
|
||||
pub mod video_thumbnail_suffixes;
|
||||
@@ -0,0 +1,16 @@
|
||||
//! Shared config for thumbnails.
|
||||
|
||||
/// NB: DO NOT CHANGE WITHOUT MIGRATION - THIS IS SHARED BETWEEN SERVICES
|
||||
/// Video media files have conventional gif thumbnails with this suffix.
|
||||
/// This is how files are physically named in the public bucket.
|
||||
pub const VIDEO_ANIMATED_GIF_THUMBNAIL_SUFFIX: &str = "-thumb.gif";
|
||||
|
||||
/// NB: DO NOT CHANGE WITHOUT MIGRATION - THIS IS SHARED BETWEEN SERVICES
|
||||
/// Video media files have conventional static jpg thumbnails with this suffix.
|
||||
/// This is how files are physically named in the public bucket.
|
||||
pub const VIDEO_STATIC_JPG_THUMBNAIL_SUFFIX: &str = "-thumb.jpg";
|
||||
|
||||
/// The current thumbnail format is a dual jpg + gif of a set size.
|
||||
/// We write this as version "1". If we change the thumbnails in the behavior,
|
||||
/// we may increment this number so that we can backfill.
|
||||
pub const CURRENT_VIDEO_THUMBNAIL_VERSION : u8 = 1;
|
||||
@@ -20,7 +20,7 @@ path = "src/lib.rs"
|
||||
workspace-build-acceleration.workspace = true
|
||||
|
||||
# Internal
|
||||
bucket_paths = { path = "../../../lib/bucket_paths" }
|
||||
bucket_paths.workspace = true
|
||||
composite_identifiers = { path = "../../../schema/public/composite_identifiers" }
|
||||
enums = { workspace = true, features = ["database"] }
|
||||
errors.workspace = true
|
||||
|
||||
@@ -2,5 +2,6 @@ pub mod create;
|
||||
pub mod delete;
|
||||
pub mod edit;
|
||||
pub mod get;
|
||||
pub mod thumbnails;
|
||||
pub mod list;
|
||||
pub mod upsert;
|
||||
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
use chrono::{DateTime, Utc};
|
||||
use enums::by_table::media_files::media_file_class::MediaFileClass;
|
||||
use sqlx::{Executor, MySql};
|
||||
use tokens::tokens::media_files::MediaFileToken;
|
||||
|
||||
const DEFAULT_PAGE_SIZE: i64 = 100;
|
||||
|
||||
const DEFAULT_MAX_LOOKBACK_HOURS: i32 = 4;
|
||||
|
||||
pub struct ListVideoMediaFilesWithoutThumbnailsArgs<E> {
|
||||
/// Override the maximum number of hours in our lookback window.
|
||||
pub custom_max_lookback_hours: Option<i32>,
|
||||
/// Override how many results to fetch (given a full page of results)
|
||||
pub custom_page_size: Option<i64>,
|
||||
/// Cursor to continue paginating with
|
||||
pub maybe_id_cursor: Option<i64>,
|
||||
/// The MySQL executor to run the query on.
|
||||
pub executor: E,
|
||||
}
|
||||
|
||||
pub struct VideoMediaFilesWithoutThumbnails {
|
||||
pub media_files: Vec<VideoMediaFileWithoutThumbnail>,
|
||||
pub next_cursor: Option<i64>,
|
||||
}
|
||||
|
||||
pub struct VideoMediaFileWithoutThumbnail {
|
||||
pub id: i64,
|
||||
pub token: MediaFileToken,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub maybe_thumbnail_version: Option<u8>,
|
||||
pub public_bucket_directory_hash: String,
|
||||
pub maybe_public_bucket_prefix: Option<String>,
|
||||
pub maybe_public_bucket_extension: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn list_video_media_files_without_thumbnails_for_job<'e, 'c, E>(
|
||||
args: ListVideoMediaFilesWithoutThumbnailsArgs<E>,
|
||||
) -> Result<VideoMediaFilesWithoutThumbnails, sqlx::Error>
|
||||
where E: 'e + Executor<'c, Database = MySql>
|
||||
{
|
||||
let cursor = args.maybe_id_cursor.unwrap_or(i64::MAX);
|
||||
|
||||
let page_size = args.custom_page_size.unwrap_or(DEFAULT_PAGE_SIZE);
|
||||
let max_lookback_hours = args.custom_max_lookback_hours.unwrap_or(DEFAULT_MAX_LOOKBACK_HOURS);
|
||||
|
||||
const MEDIA_CLASS_VIDEO: &str = MediaFileClass::Video.to_str();
|
||||
|
||||
// NB(1): `COALESCE ...` helps us force the index for a more performant query plan (otherwise it explodes into a table scan).
|
||||
// NB(2): `NOW() - INTERVAL ? HOUR` uses the database clock to avoid client/server clock skew.
|
||||
let media_files = sqlx::query_as!(
|
||||
VideoMediaFileWithoutThumbnail,
|
||||
r#"
|
||||
SELECT
|
||||
id,
|
||||
token as `token: MediaFileToken`,
|
||||
created_at as `created_at: DateTime<Utc>`,
|
||||
maybe_thumbnail_version as `maybe_thumbnail_version: u8`,
|
||||
public_bucket_directory_hash,
|
||||
maybe_public_bucket_prefix,
|
||||
maybe_public_bucket_extension
|
||||
FROM media_files
|
||||
WHERE
|
||||
id >= (SELECT COALESCE(MIN(id), 0) FROM media_files WHERE created_at >= NOW() - INTERVAL ? HOUR)
|
||||
AND id < ?
|
||||
AND media_class = ?
|
||||
AND maybe_thumbnail_version IS NULL
|
||||
AND user_deleted_at IS NULL
|
||||
AND mod_deleted_at IS NULL
|
||||
ORDER BY id DESC
|
||||
LIMIT ?
|
||||
"#,
|
||||
max_lookback_hours,
|
||||
cursor,
|
||||
MEDIA_CLASS_VIDEO,
|
||||
page_size,
|
||||
)
|
||||
.fetch_all(args.executor)
|
||||
.await?;
|
||||
|
||||
let next_cursor = if media_files.len() as i64 == page_size {
|
||||
media_files.last().map(|f| f.id)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok(VideoMediaFilesWithoutThumbnails { media_files, next_cursor })
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
pub mod list_video_media_files_without_thumbnails_for_job;
|
||||
pub mod update_video_media_file_with_thumbnail;
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
use sqlx::{Executor, MySql};
|
||||
use tokens::tokens::media_files::MediaFileToken;
|
||||
|
||||
pub async fn update_video_media_file_with_thumbnail<'e, 'c, E>(
|
||||
media_file_token: &MediaFileToken,
|
||||
thumbnail_version: u8,
|
||||
executor: E,
|
||||
) -> Result<(), sqlx::Error>
|
||||
where E: 'e + Executor<'c, Database = MySql>
|
||||
{
|
||||
sqlx::query!(
|
||||
r#"
|
||||
UPDATE media_files
|
||||
SET maybe_thumbnail_version = ?
|
||||
WHERE token = ?
|
||||
LIMIT 1
|
||||
"#,
|
||||
thumbnail_version,
|
||||
media_file_token.as_str(),
|
||||
)
|
||||
.execute(executor)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -37,7 +37,7 @@ impl_mysql_from_row!(MediaFileClass);
|
||||
|
||||
/// NB: Legacy API for older code.
|
||||
impl MediaFileClass {
|
||||
pub fn to_str(&self) -> &'static str {
|
||||
pub const fn to_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Unknown => "unknown",
|
||||
Self::Audio => "audio",
|
||||
|
||||
@@ -21,7 +21,7 @@ workspace-build-acceleration.workspace = true
|
||||
|
||||
# Internal
|
||||
bootstrap = { path = "../../../lib/bootstrap" }
|
||||
bucket_paths = { path = "../../../lib/bucket_paths" }
|
||||
bucket_paths.workspace = true
|
||||
cloud_storage.workspace = true
|
||||
concurrency = { path = "../../../lib/concurrency" }
|
||||
config = { path = "../../../lib/deprecated/config" }
|
||||
|
||||
@@ -22,7 +22,7 @@ workspace-build-acceleration.workspace = true
|
||||
# Internal
|
||||
actix_helpers = { path = "../../../lib/actix_helpers" }
|
||||
bootstrap = { path = "../../../lib/bootstrap" }
|
||||
bucket_paths = { path = "../../../lib/bucket_paths" }
|
||||
bucket_paths.workspace = true
|
||||
cloud_storage.workspace = true
|
||||
collections = { path = "../../../lib/data/collections" }
|
||||
composite_identifiers = { path = "../../../schema/public/composite_identifiers" }
|
||||
@@ -49,6 +49,7 @@ subprocess_common = { path = "../../../lib/subprocess_common" }
|
||||
thumbnail_generator = { path = "../../../api_clients/thumbnail_generator"}
|
||||
tokens.workspace = true
|
||||
tts_common = { path = "../../../lib/tts_common" }
|
||||
ffmpeg_utils.workspace = true
|
||||
videos = { path = "../../../lib/files/videos" }
|
||||
|
||||
# External
|
||||
|
||||
+1
-1
@@ -33,7 +33,7 @@ use std::time::{Duration, Instant};
|
||||
use sqlx::MySqlPool;
|
||||
use subprocess_common::command_runner::command_args::CommandArgs;
|
||||
use subprocess_common::command_runner::command_runner_args::{RunAsSubprocessArgs, StreamRedirection};
|
||||
use videos::ffprobe_get_dimensions::ffprobe_get_dimensions;
|
||||
use ffmpeg_utils::ffprobe::ffprobe_get_dimensions::ffprobe_get_dimensions;
|
||||
|
||||
enum StudioModelPipeline<'a> {
|
||||
None,
|
||||
|
||||
+1
-1
@@ -23,7 +23,7 @@ use mysql_queries::payloads::media_file_extra_info::inner_payloads::face_fusion_
|
||||
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
|
||||
use mysql_queries::queries::media_files::create::insert_media_file_from_face_fusion::{insert_media_file_from_face_fusion, InsertFaceFusionArgs};
|
||||
use thumbnail_generator::task_client::thumbnail_task::{ThumbnailTaskBuilder, ThumbnailTaskInputMimeType};
|
||||
use videos::ffprobe_get_info::ffprobe_get_info;
|
||||
use ffmpeg_utils::ffprobe::ffprobe_get_info::ffprobe_get_info;
|
||||
|
||||
use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
|
||||
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
|
||||
|
||||
+1
-1
@@ -23,7 +23,7 @@ use mysql_queries::payloads::media_file_extra_info::inner_payloads::live_portrai
|
||||
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
|
||||
use mysql_queries::queries::media_files::create::insert_media_file_from_live_portrait::{insert_media_file_from_live_portrait, InsertLivePortraitArgs};
|
||||
use thumbnail_generator::task_client::thumbnail_task::{ThumbnailTaskBuilder, ThumbnailTaskInputMimeType};
|
||||
use videos::ffprobe_get_info::ffprobe_get_info;
|
||||
use ffmpeg_utils::ffprobe::ffprobe_get_info::ffprobe_get_info;
|
||||
|
||||
use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
|
||||
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ use mysql_queries::payloads::generic_inference_args::generic_inference_args::Pol
|
||||
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
|
||||
use mysql_queries::queries::model_weights::get::get_weight::get_weight_by_token_with_transactor;
|
||||
use mysql_queries::utils::transactor::Transactor;
|
||||
use videos::ffprobe_get_dimensions::ffprobe_get_dimensions;
|
||||
use ffmpeg_utils::ffprobe::ffprobe_get_dimensions::ffprobe_get_dimensions;
|
||||
|
||||
use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
|
||||
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@ use log::{error, info};
|
||||
|
||||
use filesys::path_to_string::path_to_string;
|
||||
use mysql_queries::payloads::generic_inference_args::inner_payloads::workflow_payload::WorkflowArgs;
|
||||
use videos::ffprobe_get_dimensions::ffprobe_get_dimensions;
|
||||
use ffmpeg_utils::ffprobe::ffprobe_get_dimensions::ffprobe_get_dimensions;
|
||||
|
||||
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
|
||||
use crate::job::job_types::workflow::comfy_ui_dependencies::ComfyDependencies;
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ use log::debug;
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
use subprocess_common::command_runner::command_args::CommandArgs;
|
||||
use videos::ffmpeg_timestamp_from_duration::ffmpeg_timestamp_from_duration;
|
||||
use ffmpeg_utils::ffmpeg::ffmpeg_timestamp_from_duration::ffmpeg_timestamp_from_duration;
|
||||
|
||||
/// Trim a video's duration
|
||||
/// See: https://shotstack.io/learn/use-ffmpeg-to-trim-video/
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ use log::debug;
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
use subprocess_common::command_runner::command_args::CommandArgs;
|
||||
use videos::ffmpeg_timestamp_from_duration::ffmpeg_timestamp_from_duration;
|
||||
use ffmpeg_utils::ffmpeg::ffmpeg_timestamp_from_duration::ffmpeg_timestamp_from_duration;
|
||||
|
||||
/// Resample a video's frame rate
|
||||
/// See: https://trac.ffmpeg.org/wiki/ChangingFrameRate
|
||||
|
||||
@@ -21,12 +21,12 @@ workspace-build-acceleration = { workspace = true }
|
||||
|
||||
# Internal
|
||||
actix_helpers.workspace = true
|
||||
bootstrap = { path = "../../../lib/bootstrap" }
|
||||
bucket_paths = { path = "../../../lib/bucket_paths" }
|
||||
bootstrap.workspace = true
|
||||
bucket_paths.workspace = true
|
||||
cloud_storage.workspace = true
|
||||
concurrency = { path = "../../../lib/concurrency" }
|
||||
concurrency.workspace = true
|
||||
config = { path = "../../../lib/deprecated/config" }
|
||||
easyenv = { path = "../../../lib/easyenv" }
|
||||
easyenv.workspace = true
|
||||
enums.workspace = true
|
||||
errors.workspace = true
|
||||
hashing = { path = "../../../lib/files/hashing" }
|
||||
|
||||
@@ -18,7 +18,7 @@ normal = ["workspace-build-acceleration"]
|
||||
[dependencies]
|
||||
|
||||
# Internal
|
||||
bucket_paths = { path = "../../../lib/bucket_paths" }
|
||||
bucket_paths.workspace = true
|
||||
cloud_storage.workspace = true
|
||||
config = { path = "../../../lib/deprecated/config" }
|
||||
easyenv = { path = "../../../lib/easyenv" }
|
||||
|
||||
@@ -20,7 +20,7 @@ normal = ["workspace-build-acceleration"]
|
||||
workspace-build-acceleration.workspace = true
|
||||
|
||||
# Internal
|
||||
bucket_paths = { path = "../../../lib/bucket_paths" }
|
||||
bucket_paths.workspace = true
|
||||
cloud_storage.workspace = true
|
||||
collections = { path = "../../../lib/data/collections" }
|
||||
config = { path = "../../../lib/deprecated/config" }
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
[package]
|
||||
name = "video-thumbnail-job"
|
||||
edition = "2021"
|
||||
version = "0.0.1"
|
||||
authors = [
|
||||
"Brandon Thomas <bt@brand.io>",
|
||||
"Brandon Thomas <echelon@gmail.com>",
|
||||
]
|
||||
publish = false
|
||||
|
||||
[[bin]]
|
||||
name = "video-thumbnail-job"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.cargo-udeps.ignore]
|
||||
normal = ["workspace-build-acceleration"]
|
||||
|
||||
[dependencies]
|
||||
# cargo-hakari workspace hack (faster compile times)
|
||||
workspace-build-acceleration.workspace = true
|
||||
|
||||
# Internal
|
||||
actix_helpers.workspace = true
|
||||
bootstrap.workspace = true
|
||||
bucket_paths.workspace = true
|
||||
cloud_storage.workspace = true
|
||||
ffmpeg_utils.workspace = true
|
||||
concurrency.workspace = true
|
||||
config = { path = "../../../lib/deprecated/config" }
|
||||
easyenv.workspace = true
|
||||
errors.workspace = true
|
||||
jobs_common.workspace = true
|
||||
mysql_queries.workspace = true
|
||||
server_environment = { path = "../../../lib/server_environment" }
|
||||
|
||||
# External
|
||||
actix-web.workspace = true
|
||||
anyhow.workspace = true
|
||||
log.workspace = true
|
||||
serde.workspace = true
|
||||
serde_derive.workspace = true
|
||||
serde_json.workspace = true
|
||||
sqlx = { version = "0.7.4", features = [ "mysql", "runtime-tokio-rustls", "chrono" ] }
|
||||
tempdir.workspace = true
|
||||
tokio.workspace = true
|
||||
@@ -0,0 +1,4 @@
|
||||
# default .env configurations (these can be overridden with k8s)
|
||||
|
||||
# Development
|
||||
SERVER_ENVIRONMENT='development'
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use actix_web::error::ResponseError;
|
||||
use actix_web::http::header::ContentType;
|
||||
use actix_web::http::StatusCode;
|
||||
use actix_web::{web, HttpRequest, HttpResponse};
|
||||
use log::error;
|
||||
|
||||
use actix_helpers::response_serializers::error_to_json_http_response::error_to_json_http_response;
|
||||
|
||||
use crate::http_server::http_server_shared_state::HttpServerSharedState;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct HealthCheckResponse {
|
||||
pub success: bool,
|
||||
pub is_healthy: bool,
|
||||
pub consecutive_failure_count: u64,
|
||||
pub consecutive_success_count: u64,
|
||||
pub total_failure_count: u64,
|
||||
pub total_success_count: u64,
|
||||
pub total_failure_ratio: f32,
|
||||
pub total_success_ratio: f32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub enum HealthCheckError {
|
||||
ServerError,
|
||||
}
|
||||
|
||||
impl ResponseError for HealthCheckError {
|
||||
fn status_code(&self) -> StatusCode {
|
||||
match *self {
|
||||
HealthCheckError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
}
|
||||
}
|
||||
|
||||
fn error_response(&self) -> HttpResponse {
|
||||
error_to_json_http_response(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for HealthCheckError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{:?}", self)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_health_check_handler(
|
||||
_http_request: HttpRequest,
|
||||
server_state: web::Data<Arc<HttpServerSharedState>>,
|
||||
) -> Result<HttpResponse, HealthCheckError> {
|
||||
let job_stats = server_state
|
||||
.job_stats
|
||||
.get_status()
|
||||
.map_err(|e| {
|
||||
error!("Error serving health check status: {:?}", e);
|
||||
HealthCheckError::ServerError
|
||||
})?;
|
||||
|
||||
let total_tries = job_stats
|
||||
.total_failure_count
|
||||
.saturating_add(job_stats.total_success_count);
|
||||
|
||||
let total_success_ratio = if total_tries > 0 {
|
||||
(job_stats.total_success_count as f32) / (total_tries as f32)
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
let total_failure_ratio = if total_tries > 0 {
|
||||
1.0 - total_success_ratio
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
let is_healthy =
|
||||
job_stats.consecutive_failure_count < server_state.consecutive_failure_unhealthy_threshold;
|
||||
|
||||
let response = HealthCheckResponse {
|
||||
success: true,
|
||||
is_healthy,
|
||||
consecutive_failure_count: job_stats.consecutive_failure_count,
|
||||
consecutive_success_count: job_stats.consecutive_success_count,
|
||||
total_failure_count: job_stats.total_failure_count,
|
||||
total_success_count: job_stats.total_success_count,
|
||||
total_failure_ratio,
|
||||
total_success_ratio,
|
||||
};
|
||||
|
||||
let body = serde_json::to_string(&response).map_err(|_e| HealthCheckError::ServerError)?;
|
||||
|
||||
if is_healthy {
|
||||
Ok(HttpResponse::Ok()
|
||||
.content_type(ContentType::json())
|
||||
.body(body))
|
||||
} else {
|
||||
Ok(HttpResponse::InternalServerError()
|
||||
.content_type(ContentType::json())
|
||||
.body(body))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
pub mod health_check_handler;
|
||||
@@ -0,0 +1,7 @@
|
||||
use jobs_common::job_stats::JobStats;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct HttpServerSharedState {
|
||||
pub job_stats: JobStats,
|
||||
pub consecutive_failure_unhealthy_threshold: u64,
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
pub mod endpoints;
|
||||
pub mod http_server_shared_state;
|
||||
pub mod run_http_server;
|
||||
@@ -0,0 +1,67 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use actix_web::dev::Server;
|
||||
use actix_web::middleware::Logger;
|
||||
use actix_web::{web, App, HttpResponse, HttpServer};
|
||||
use log::info;
|
||||
|
||||
use bootstrap::bootstrap::ContainerEnvironment;
|
||||
use errors::AnyhowResult;
|
||||
use jobs_common::job_stats::JobStats;
|
||||
|
||||
use crate::http_server::endpoints::health_check_handler::get_health_check_handler;
|
||||
use crate::http_server::http_server_shared_state::HttpServerSharedState;
|
||||
|
||||
const DEFAULT_BIND_ADDRESS: &str = "0.0.0.0:12345";
|
||||
const DEFAULT_NUM_WORKERS: usize = 2;
|
||||
|
||||
pub struct CreateServerArgs {
|
||||
pub container_environment: ContainerEnvironment,
|
||||
pub job_stats: JobStats,
|
||||
}
|
||||
|
||||
pub fn run_http_server(args: CreateServerArgs) -> AnyhowResult<Server> {
|
||||
let bind_address = easyenv::get_env_string_or_default("HTTP_BIND_ADDRESS", DEFAULT_BIND_ADDRESS);
|
||||
let num_workers = easyenv::get_env_num("HTTP_NUM_WORKERS", DEFAULT_NUM_WORKERS)?;
|
||||
let _hostname = args.container_environment.hostname.clone();
|
||||
|
||||
let server_state = HttpServerSharedState {
|
||||
job_stats: args.job_stats.clone(),
|
||||
consecutive_failure_unhealthy_threshold: easyenv::get_env_num(
|
||||
"CONSECUTIVE_FAILURE_UNHEALTHY_THRESHOLD",
|
||||
3,
|
||||
)?,
|
||||
};
|
||||
|
||||
let server_state_arc = web::Data::new(Arc::new(server_state));
|
||||
|
||||
info!("Starting HTTP service (for k8s health checking).");
|
||||
|
||||
let log_format = "[%{HOSTNAME}e] IP=[%{X-Forwarded-For}i] \"%r\" %s %b \"%{Referer}i\" \"%{User-Agent}i\" %T";
|
||||
|
||||
let handle = HttpServer::new(move || {
|
||||
App::new()
|
||||
.app_data(server_state_arc.clone())
|
||||
.wrap(Logger::new(&log_format).exclude("/_status"))
|
||||
.service(
|
||||
web::resource("/")
|
||||
.route(web::get().to(|| HttpResponse::Ok()))
|
||||
.route(web::head().to(|| HttpResponse::Ok())),
|
||||
)
|
||||
.service(
|
||||
web::resource("/_status")
|
||||
.route(web::get().to(get_health_check_handler))
|
||||
.route(web::head().to(|| HttpResponse::Ok())),
|
||||
)
|
||||
})
|
||||
.bind(bind_address)?
|
||||
.workers(num_workers)
|
||||
.run();
|
||||
|
||||
Ok(handle)
|
||||
}
|
||||
|
||||
pub async fn launch_http_server(args: CreateServerArgs) -> AnyhowResult<()> {
|
||||
run_http_server(args)?.await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use log::{error, info, warn};
|
||||
|
||||
use mysql_queries::queries::media_files::thumbnails::list_video_media_files_without_thumbnails_for_job::{
|
||||
list_video_media_files_without_thumbnails_for_job,
|
||||
ListVideoMediaFilesWithoutThumbnailsArgs,
|
||||
};
|
||||
|
||||
use crate::job_dependencies::JobDependencies;
|
||||
use crate::job::process_single_media_file::process_single_media_file;
|
||||
|
||||
pub async fn main_loop(deps: JobDependencies) {
|
||||
while !deps.application_shutdown.get() {
|
||||
let processed_any = match run_batch_cycle(&deps).await {
|
||||
Ok(count) => {
|
||||
if count > 0 {
|
||||
info!("Processed {} video thumbnail(s) this cycle.", count);
|
||||
}
|
||||
count > 0
|
||||
}
|
||||
Err(err) => {
|
||||
error!("Error in video thumbnail batch cycle: {:?}", err);
|
||||
let _ = deps.job_stats.increment_failure_count();
|
||||
|
||||
// Wait before retrying after a failure.
|
||||
tokio::time::sleep(Duration::from_millis(deps.query_failure_retry_delay_millis)).await;
|
||||
false
|
||||
}
|
||||
};
|
||||
|
||||
// If we didn't process anything, sleep for the poll interval before checking again.
|
||||
if !processed_any {
|
||||
tokio::time::sleep(Duration::from_millis(deps.poll_interval_millis)).await;
|
||||
}
|
||||
}
|
||||
|
||||
warn!("Video thumbnail job main loop is shut down.");
|
||||
}
|
||||
|
||||
/// Run one full pagination cycle: keep querying pages of media files without thumbnails
|
||||
/// until there are no more results. Returns the total number of items processed.
|
||||
async fn run_batch_cycle(deps: &JobDependencies) -> anyhow::Result<u64> {
|
||||
let mut maybe_cursor: Option<i64> = None;
|
||||
let mut total_processed: u64 = 0;
|
||||
|
||||
loop {
|
||||
if deps.application_shutdown.get() {
|
||||
break;
|
||||
}
|
||||
|
||||
let result = list_video_media_files_without_thumbnails_for_job(
|
||||
ListVideoMediaFilesWithoutThumbnailsArgs {
|
||||
custom_max_lookback_hours: deps.custom_max_lookback_hours,
|
||||
custom_page_size: deps.custom_page_size,
|
||||
maybe_id_cursor: maybe_cursor,
|
||||
executor: &deps.mysql_pool,
|
||||
},
|
||||
).await?;
|
||||
|
||||
if result.media_files.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
for media_file in &result.media_files {
|
||||
if deps.application_shutdown.get() {
|
||||
break;
|
||||
}
|
||||
|
||||
match process_single_media_file(deps, media_file).await {
|
||||
Ok(()) => {
|
||||
let _ = deps.job_stats.increment_success_count();
|
||||
total_processed += 1;
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"Failed to generate thumbnail for media file {}: {:?}",
|
||||
media_file.token.as_str(),
|
||||
err,
|
||||
);
|
||||
let _ = deps.job_stats.increment_failure_count();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
maybe_cursor = result.next_cursor;
|
||||
if maybe_cursor.is_none() {
|
||||
break;
|
||||
}
|
||||
|
||||
// Throttle between pages.
|
||||
tokio::time::sleep(Duration::from_millis(deps.query_delay_millis)).await;
|
||||
}
|
||||
|
||||
Ok(total_processed)
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
pub mod main_loop;
|
||||
pub mod process_single_media_file;
|
||||
@@ -0,0 +1,135 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use log::info;
|
||||
use tempdir::TempDir;
|
||||
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
use bucket_paths::path_conventions::video_thumbnail_suffixes::{CURRENT_VIDEO_THUMBNAIL_VERSION, VIDEO_ANIMATED_GIF_THUMBNAIL_SUFFIX, VIDEO_STATIC_JPG_THUMBNAIL_SUFFIX};
|
||||
use ffmpeg_utils::ffmpeg::ffmpeg_video_first_frame_to_jpg_thumbnail::{ffmpeg_video_first_frame_to_jpg_thumbnail, FfmpegVideoFirstFrameToJpgThumbnailArgs};
|
||||
use ffmpeg_utils::ffmpeg::ffmpeg_video_gif_preview::{ffmpeg_video_gif_preview, FfmpegVideoGifPreviewArgs};
|
||||
use mysql_queries::queries::media_files::thumbnails::list_video_media_files_without_thumbnails_for_job::VideoMediaFileWithoutThumbnail;
|
||||
use mysql_queries::queries::media_files::thumbnails::update_video_media_file_with_thumbnail::update_video_media_file_with_thumbnail;
|
||||
|
||||
use crate::job_dependencies::JobDependencies;
|
||||
|
||||
/// A downloaded video file alongside its owning temp directory.
|
||||
/// The temp directory (and its contents) are cleaned up when this struct is dropped.
|
||||
pub struct DownloadedFile {
|
||||
pub temp_dir: TempDir,
|
||||
pub file_path: PathBuf,
|
||||
}
|
||||
|
||||
/// Download the source video from the bucket, generate thumbnails, and upload them.
|
||||
pub async fn process_single_media_file(
|
||||
deps: &JobDependencies,
|
||||
media_file: &VideoMediaFileWithoutThumbnail,
|
||||
) -> anyhow::Result<()> {
|
||||
let downloaded = download_video(deps, media_file).await?;
|
||||
|
||||
info!(
|
||||
"Downloaded video to {:?}. Generating thumbnails for {:?}.",
|
||||
downloaded.file_path,
|
||||
media_file.token,
|
||||
);
|
||||
|
||||
// Upload thumbnails to bucket beside the original video
|
||||
let video_object_path = get_video_object_path(media_file);
|
||||
|
||||
// Generate jpg thumbnail
|
||||
let jpg_path = downloaded.temp_dir.path().join("thumbnail.jpg");
|
||||
|
||||
ffmpeg_video_first_frame_to_jpg_thumbnail(
|
||||
FfmpegVideoFirstFrameToJpgThumbnailArgs {
|
||||
input_video_path: &downloaded.file_path,
|
||||
output_jpg_path: &jpg_path,
|
||||
},
|
||||
)?;
|
||||
|
||||
info!("Generated JPG thumbnail for {}", media_file.token.as_str());
|
||||
|
||||
let jpg_object_path = format!("{video_object_path}{VIDEO_STATIC_JPG_THUMBNAIL_SUFFIX}");
|
||||
|
||||
deps
|
||||
.public_bucket_client
|
||||
.upload_filename(&jpg_object_path, &jpg_path)
|
||||
.await?;
|
||||
|
||||
info!("Uploaded JPG thumbnail to {}", jpg_object_path);
|
||||
|
||||
// Generate gif thumbnail
|
||||
let gif_path = downloaded.temp_dir.path().join("thumbnail.gif");
|
||||
|
||||
ffmpeg_video_gif_preview(
|
||||
FfmpegVideoGifPreviewArgs {
|
||||
input_video_path: &downloaded.file_path,
|
||||
output_gif_path: &gif_path,
|
||||
},
|
||||
)?;
|
||||
|
||||
info!("Generated GIF preview for {}", media_file.token.as_str());
|
||||
|
||||
let gif_object_path = format!("{video_object_path}{VIDEO_ANIMATED_GIF_THUMBNAIL_SUFFIX}");
|
||||
|
||||
deps
|
||||
.public_bucket_client
|
||||
.upload_filename(&gif_object_path, &gif_path)
|
||||
.await?;
|
||||
|
||||
info!("Uploaded GIF preview to {}", gif_object_path);
|
||||
|
||||
info!("Marking thumbnail job for {:?} done", media_file.token);
|
||||
|
||||
// Mark the media file as having a thumbnail in the database.
|
||||
update_video_media_file_with_thumbnail(
|
||||
&media_file.token,
|
||||
CURRENT_VIDEO_THUMBNAIL_VERSION,
|
||||
&deps.mysql_pool,
|
||||
).await?;
|
||||
|
||||
info!("Updated thumbnail version for {}", media_file.token.as_str());
|
||||
|
||||
// `downloaded.temp_dir` is dropped here, cleaning up the temp directory and all contents.
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Build the full bucket object path for the video file.
|
||||
fn get_video_object_path(media_file: &VideoMediaFileWithoutThumbnail) -> String {
|
||||
let bucket_path = MediaFileBucketPath::from_object_hash(
|
||||
&media_file.public_bucket_directory_hash,
|
||||
media_file.maybe_public_bucket_prefix.as_deref(),
|
||||
media_file.maybe_public_bucket_extension.as_deref(),
|
||||
);
|
||||
|
||||
bucket_path.get_full_object_path_str().to_string()
|
||||
}
|
||||
|
||||
/// Download the source video from the public bucket into a new temp directory.
|
||||
async fn download_video(
|
||||
deps: &JobDependencies,
|
||||
media_file: &VideoMediaFileWithoutThumbnail,
|
||||
) -> anyhow::Result<DownloadedFile> {
|
||||
let object_path = get_video_object_path(media_file);
|
||||
|
||||
info!(
|
||||
"Downloading video for media file {} from bucket path: {}",
|
||||
media_file.token.as_str(),
|
||||
object_path,
|
||||
);
|
||||
|
||||
let temp_dir = TempDir::new_in(&deps.temp_dir, "video_thumbnail")?;
|
||||
|
||||
let video_extension = media_file
|
||||
.maybe_public_bucket_extension
|
||||
.as_deref()
|
||||
.unwrap_or(".mp4");
|
||||
|
||||
let filename = format!("{}{}", media_file.token.as_str(), video_extension);
|
||||
let file_path = temp_dir.path().join(&filename);
|
||||
|
||||
deps
|
||||
.public_bucket_client
|
||||
.download_file_to_disk(&object_path, &file_path)
|
||||
.await?;
|
||||
|
||||
Ok(DownloadedFile { temp_dir, file_path })
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
|
||||
use jobs_common::job_stats::JobStats;
|
||||
use server_environment::ServerEnvironment;
|
||||
use sqlx::MySqlPool;
|
||||
|
||||
pub struct JobDependencies {
|
||||
pub mysql_pool: MySqlPool,
|
||||
|
||||
/// Public GCS/S3 bucket for downloading source videos and uploading generated thumbnails.
|
||||
pub public_bucket_client: BucketClient,
|
||||
|
||||
pub server_environment: ServerEnvironment,
|
||||
|
||||
pub job_stats: JobStats,
|
||||
|
||||
/// How long to sleep between poll iterations when there is no work (milliseconds).
|
||||
pub poll_interval_millis: u64,
|
||||
|
||||
/// Minimum delay between successive database queries within a single poll cycle (milliseconds).
|
||||
pub query_delay_millis: u64,
|
||||
|
||||
/// How long to wait after a query failure before retrying (milliseconds).
|
||||
pub query_failure_retry_delay_millis: u64,
|
||||
|
||||
/// Override for the lookback window (hours). `None` uses the query default.
|
||||
pub custom_max_lookback_hours: Option<i32>,
|
||||
|
||||
/// Override for the query page size. `None` uses the query default.
|
||||
pub custom_page_size: Option<i64>,
|
||||
|
||||
/// Root directory for temporary files (video downloads, thumbnail intermediates).
|
||||
pub temp_dir: PathBuf,
|
||||
|
||||
/// Set to `true` from another thread to trigger graceful shutdown.
|
||||
pub application_shutdown: RelaxedAtomicBool,
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
// Never allow these
|
||||
#![forbid(private_bounds)]
|
||||
#![forbid(private_interfaces)]
|
||||
#![forbid(unused_must_use)]
|
||||
|
||||
// Always allow
|
||||
#![allow(dead_code)]
|
||||
#![allow(non_snake_case)]
|
||||
|
||||
#[macro_use] extern crate serde_derive;
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::anyhow;
|
||||
use log::{info, warn};
|
||||
use sqlx::mysql::MySqlPoolOptions;
|
||||
|
||||
use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
|
||||
use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
|
||||
use config::shared_constants::DEFAULT_RUST_LOG;
|
||||
use errors::AnyhowResult;
|
||||
use jobs_common::job_stats::JobStats;
|
||||
use server_environment::ServerEnvironment;
|
||||
|
||||
use crate::http_server::run_http_server::{launch_http_server, CreateServerArgs};
|
||||
use crate::job::main_loop::main_loop;
|
||||
use crate::job_dependencies::JobDependencies;
|
||||
|
||||
pub mod http_server;
|
||||
pub mod job;
|
||||
pub mod job_dependencies;
|
||||
|
||||
// Bucket config
|
||||
const ENV_ACCESS_KEY: &str = "ACCESS_KEY";
|
||||
const ENV_SECRET_KEY: &str = "SECRET_KEY";
|
||||
const ENV_REGION_NAME: &str = "REGION_NAME";
|
||||
const ENV_PUBLIC_BUCKET_NAME: &str = "PUBLIC_BUCKET_NAME";
|
||||
const ENV_S3_ENDPOINT: &str = "S3_COMPATIBLE_ENDPOINT_URL";
|
||||
|
||||
#[actix_web::main]
|
||||
async fn main() -> AnyhowResult<()> {
|
||||
|
||||
let container_environment = bootstrap(BootstrapArgs {
|
||||
app_name: "video-thumbnail-job",
|
||||
default_logging_override: Some(DEFAULT_RUST_LOG),
|
||||
config_search_directories: &[".", "./config", "crates/service/job/video_thumbnail_job/config"],
|
||||
ignore_legacy_dot_env_file: true,
|
||||
})?;
|
||||
|
||||
info!("Hostname: {}", &container_environment.hostname);
|
||||
|
||||
let _k8s_node_name = easyenv::get_env_string_optional("K8S_NODE_NAME");
|
||||
let _k8s_pod_name = easyenv::get_env_string_optional("K8S_POD_NAME");
|
||||
|
||||
let db_connection_string = easyenv::get_env_string_or_default(
|
||||
"MYSQL_URL",
|
||||
DEFAULT_MYSQL_CONNECTION_STRING,
|
||||
);
|
||||
|
||||
info!("Connecting to database...");
|
||||
|
||||
let mysql_pool = MySqlPoolOptions::new()
|
||||
.max_connections(2)
|
||||
.connect(&db_connection_string)
|
||||
.await?;
|
||||
|
||||
info!("Connected to MySQL.");
|
||||
|
||||
let server_environment = ServerEnvironment::from_str(
|
||||
&easyenv::get_env_string_required("SERVER_ENVIRONMENT")?,
|
||||
)
|
||||
.ok_or(anyhow!("invalid server environment"))?;
|
||||
|
||||
// Bucket setup
|
||||
let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?;
|
||||
let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?;
|
||||
let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?;
|
||||
let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?;
|
||||
let s3_compatible_endpoint_url = easyenv::get_env_string_required(ENV_S3_ENDPOINT)?;
|
||||
|
||||
let bucket_timeout = easyenv::get_env_duration_seconds_or_default(
|
||||
"BUCKET_TIMEOUT_SECONDS",
|
||||
Duration::from_secs(60 * 5),
|
||||
);
|
||||
|
||||
let public_bucket_client = BucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
&public_bucket_name,
|
||||
&s3_compatible_endpoint_url,
|
||||
None,
|
||||
Some(bucket_timeout),
|
||||
)?;
|
||||
|
||||
// Job polling and timing configuration
|
||||
let poll_interval_millis: u64 = easyenv::get_env_num(
|
||||
"VIDEO_THUMBNAIL_POLL_INTERVAL_MILLIS",
|
||||
30_000,
|
||||
)?;
|
||||
|
||||
let query_delay_millis: u64 = easyenv::get_env_num(
|
||||
"VIDEO_THUMBNAIL_QUERY_DELAY_MILLIS",
|
||||
100,
|
||||
)?;
|
||||
|
||||
let query_failure_retry_delay_millis: u64 = easyenv::get_env_num(
|
||||
"VIDEO_THUMBNAIL_QUERY_FAILURE_RETRY_DELAY_MILLIS",
|
||||
10_000,
|
||||
)?;
|
||||
|
||||
// Optional overrides for query parameters
|
||||
let custom_max_lookback_hours: Option<i32> = easyenv::get_env_string_optional(
|
||||
"VIDEO_THUMBNAIL_MAX_LOOKBACK_HOURS",
|
||||
).and_then(|s| s.parse().ok());
|
||||
|
||||
let custom_page_size: Option<i64> = easyenv::get_env_string_optional(
|
||||
"VIDEO_THUMBNAIL_PAGE_SIZE",
|
||||
).and_then(|s| s.parse().ok());
|
||||
|
||||
// Temp directory for video downloads and thumbnail intermediates
|
||||
let temp_dir: PathBuf = easyenv::get_env_pathbuf_or_default(
|
||||
"VIDEO_THUMBNAIL_TEMP_DIR",
|
||||
"/tmp/video-thumbnails",
|
||||
);
|
||||
|
||||
// Ensure the temp directory exists
|
||||
tokio::fs::create_dir_all(&temp_dir).await?;
|
||||
|
||||
let application_shutdown = RelaxedAtomicBool::new(false);
|
||||
let job_stats = JobStats::new();
|
||||
|
||||
let create_server_args = CreateServerArgs {
|
||||
container_environment: container_environment.clone(),
|
||||
job_stats: job_stats.clone(),
|
||||
};
|
||||
|
||||
let job_dependencies = JobDependencies {
|
||||
mysql_pool,
|
||||
public_bucket_client,
|
||||
server_environment,
|
||||
job_stats,
|
||||
poll_interval_millis,
|
||||
query_delay_millis,
|
||||
query_failure_retry_delay_millis,
|
||||
custom_max_lookback_hours,
|
||||
custom_page_size,
|
||||
temp_dir,
|
||||
application_shutdown: application_shutdown.clone(),
|
||||
};
|
||||
|
||||
std::thread::spawn(move || {
|
||||
let actix_runtime = actix_web::rt::System::new();
|
||||
let http_server_handle = launch_http_server(create_server_args);
|
||||
|
||||
actix_runtime.block_on(http_server_handle)
|
||||
.expect("HTTP server should not exit.");
|
||||
|
||||
warn!("Server thread is shut down.");
|
||||
application_shutdown.set(true);
|
||||
});
|
||||
|
||||
main_loop(job_dependencies).await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -20,7 +20,7 @@ normal = ["workspace-build-acceleration"]
|
||||
workspace-build-acceleration.workspace = true
|
||||
|
||||
# Internal
|
||||
bucket_paths = { path = "../../../lib/bucket_paths" }
|
||||
bucket_paths.workspace = true
|
||||
cloud_storage = { path = "../../../api_clients/cloud_storage" }
|
||||
config = { path = "../../../lib/deprecated/config" }
|
||||
easyenv = { path = "../../../lib/easyenv" }
|
||||
|
||||
@@ -21,7 +21,7 @@ normal = ["workspace-build-acceleration"]
|
||||
workspace-build-acceleration.workspace = true
|
||||
|
||||
# Internal
|
||||
bucket_paths = { path = "../../../lib/bucket_paths" }
|
||||
bucket_paths.workspace = true
|
||||
cloud_storage.workspace = true
|
||||
config = { path = "../../../lib/deprecated/config" }
|
||||
easyenv = { path = "../../../lib/easyenv" }
|
||||
|
||||
@@ -21,12 +21,12 @@ workspace-build-acceleration = { workspace = true }
|
||||
|
||||
# Internal
|
||||
actix_helpers.workspace = true
|
||||
bootstrap = { path = "../../../lib/bootstrap" }
|
||||
bucket_paths = { path = "../../../lib/bucket_paths" }
|
||||
bootstrap.workspace = true
|
||||
bucket_paths.workspace = true
|
||||
cloud_storage.workspace = true
|
||||
concurrency = { path = "../../../lib/concurrency" }
|
||||
concurrency.workspace = true
|
||||
config = { path = "../../../lib/deprecated/config" }
|
||||
easyenv = { path = "../../../lib/easyenv" }
|
||||
easyenv.workspace = true
|
||||
enums.workspace = true
|
||||
errors.workspace = true
|
||||
hashing = { path = "../../../lib/files/hashing" }
|
||||
|
||||
@@ -37,8 +37,8 @@ actix_cors_configs = { path = "../../../lib/actix_cors_configs" }
|
||||
actix_helpers.workspace = true
|
||||
artcraft_api_defs.workspace = true
|
||||
artcraft_router.workspace = true
|
||||
bootstrap = { path = "../../../lib/bootstrap" }
|
||||
bucket_paths = { path = "../../../lib/bucket_paths" }
|
||||
bootstrap.workspace = true
|
||||
bucket_paths.workspace = true
|
||||
cloud_storage.workspace = true
|
||||
collections.workspace = true
|
||||
component_traits.workspace = true
|
||||
@@ -47,7 +47,7 @@ config = { path = "../../../lib/deprecated/config" }
|
||||
cookies.workspace = true
|
||||
crockford = { path = "../../../lib/data/crockford" }
|
||||
datetimes = { path = "../../../lib/data/datetimes" }
|
||||
easyenv = { path = "../../../lib/easyenv" }
|
||||
easyenv.workspace = true
|
||||
elasticsearch_schema = { path = "../../../schema/database/elasticsearch_schema" }
|
||||
email_sender= { path = "../../../lib/email_sender" }
|
||||
enums.workspace = true
|
||||
@@ -65,7 +65,7 @@ media = { path = "../../../lib/files/media" }
|
||||
memory_caching = { path = "../../../lib/caching/memory_caching" }
|
||||
migration = { path = "../../../lib/migration" }
|
||||
mimetypes.workspace = true
|
||||
mysql_queries = { path = "../../../schema/database/mysql_queries" }
|
||||
mysql_queries.workspace = true
|
||||
password = { path = "../../../lib/password" }
|
||||
primitives = { path = "../../../lib/data/primitives" }
|
||||
prompt_engineering = { path = "../../../lib/machine_learning/prompt_engineering" }
|
||||
@@ -82,6 +82,7 @@ url_config = { path = "../../../lib/url_config" }
|
||||
url_utils = { path = "../../../lib/url_utils" }
|
||||
user_input_common = { path = "../../../lib/user_input_common" }
|
||||
users.workspace = true
|
||||
ffmpeg_utils.workspace = true
|
||||
videos = { path = "../../../lib/files/videos" }
|
||||
world_labs_api.workspace = true
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user