Compare commits

..
12 changed files with 358 additions and 54 deletions
@@ -0,0 +1,134 @@
use anyhow::bail;
use log::{error, info};
use std::io::Write;
use std::path::Path;
use std::process::Command;
use errors::AnyhowResult;
pub struct FfmpegTranscodeToMp4Args<'a> {
pub video_input_path: &'a Path,
pub video_output_path: &'a Path,
}
/// Transcode an arbitrary input video into a streaming-friendly H.264 / AAC
/// MP4 file.
///
/// Output container: MP4 (faststart-enabled so the moov atom lives at the
/// front, which makes the file usable by HTTP range-request video players
/// before the whole thing has downloaded).
///
/// Output streams:
/// - video: H.264 (libx264), yuv420p, CRF 23, "fast" preset
/// - audio: AAC (libfdk-aac unavailable in stock ffmpeg → libmp3lame? no →
/// plain `aac`), 128 kbps
///
/// Audio is transcoded rather than copied so the output is consistent
/// regardless of the input container's audio codec.
pub fn ffmpeg_transcode_to_mp4(args: FfmpegTranscodeToMp4Args) -> AnyhowResult<()> {
let mut command = Command::new("ffmpeg");
command
.arg("-nostdin")
.arg("-y")
.arg("-i").arg(args.video_input_path)
// Video: H.264, broadly compatible chroma subsampling.
.arg("-c:v").arg("libx264")
.arg("-preset").arg("fast")
.arg("-crf").arg("23")
.arg("-pix_fmt").arg("yuv420p")
// Audio: AAC at 128 kbps.
.arg("-c:a").arg("aac")
.arg("-b:a").arg("128k")
// MP4 container with the moov atom moved to the front for fast HTTP streaming.
.arg("-movflags").arg("+faststart")
.arg(args.video_output_path);
info!("Calling ffmpeg (transcode to mp4)...");
let output = command.output()?;
if !output.status.success() {
error!("bad exit status: {}", output.status);
let _r = std::io::stdout().write_all(&output.stdout);
let _r = std::io::stderr().write_all(&output.stderr);
bail!("ffmpeg transcode to mp4 failed: {:?}", output.status.to_string());
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use tempdir::TempDir;
use test_utils::test_file_path::test_file_path;
#[test]
fn test_transcode_mov_to_mp4() {
let input_path = test_file_path("test_data/video/mov/majoras_mask_intro.mov")
.expect("test video should exist");
let temp_dir = TempDir::new_in("/tmp", "ffmpeg_transcode_to_mp4_test")
.expect("should create temp dir");
let output_path = temp_dir.path().join("transcoded.mp4");
ffmpeg_transcode_to_mp4(FfmpegTranscodeToMp4Args {
video_input_path: &input_path,
video_output_path: &output_path,
}).expect("ffmpeg should succeed");
// ── Output file exists and is non-empty ───────────────────────────────
assert!(output_path.exists(), "output mp4 should exist");
let metadata = std::fs::metadata(&output_path).expect("should stat output");
assert!(metadata.len() > 0, "output mp4 should be non-empty");
// ── ffprobe the output and assert codecs / container ──────────────────
let probe = ffprobe::ffprobe(&output_path)
.expect("ffprobe should succeed on the transcoded file");
// Container is mp4 — ffprobe reports it as either "mov,mp4,m4a,3gp,3g2,mj2"
// or "mp4" depending on version. Both contain "mp4".
let format_name = &probe.format.format_name;
assert!(
format_name.contains("mp4"),
"expected mp4 container, got format_name = {:?}",
format_name,
);
// Video stream is H.264.
let video_stream = probe.streams.iter()
.find(|s| s.codec_type.as_deref() == Some("video"))
.expect("should have a video stream");
assert_eq!(
video_stream.codec_name.as_deref(),
Some("h264"),
"expected H.264 video, got {:?}",
video_stream.codec_name,
);
assert_eq!(
video_stream.pix_fmt.as_deref(),
Some("yuv420p"),
"expected yuv420p pixel format, got {:?}",
video_stream.pix_fmt,
);
// Audio stream is AAC (if the source has audio — Majora's Mask intro does).
let audio_stream = probe.streams.iter()
.find(|s| s.codec_type.as_deref() == Some("audio"))
.expect("should have an audio stream");
assert_eq!(
audio_stream.codec_name.as_deref(),
Some("aac"),
"expected AAC audio, got {:?}",
audio_stream.codec_name,
);
// Sanity-check that the video has non-zero dimensions / duration.
assert!(video_stream.width.unwrap_or(0) > 0, "video should have width > 0");
assert!(video_stream.height.unwrap_or(0) > 0, "video should have height > 0");
}
}
@@ -1,4 +1,5 @@
pub mod ffmpeg_timestamp_from_duration;
pub mod ffmpeg_transcode_to_mp4;
pub mod ffmpeg_trim_and_resample;
pub mod ffmpeg_video_first_frame_to_jpg_thumbnail;
pub mod ffmpeg_video_gif_preview;
@@ -63,7 +63,11 @@ mod tests {
media_file_token: None,
inference_job_token: None,
third_party_id: None,
request_ip_address: None,
avt_cookie_token: None,
session_token: None,
maybe_error: None,
session_user_token: None,
}
}
@@ -223,7 +227,11 @@ mod tests {
media_file_token: None,
inference_job_token: None,
third_party_id: None,
request_ip_address: None,
avt_cookie_token: None,
session_token: None,
maybe_error: None,
session_user_token: None,
}
}
@@ -17,6 +17,7 @@ use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path:
use enums::by_table::media_files::media_file_class::MediaFileClass;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::visibility::Visibility;
use ffmpeg_utils::ffmpeg::ffmpeg_transcode_to_mp4::{ffmpeg_transcode_to_mp4, FfmpegTranscodeToMp4Args};
use ffmpeg_utils::ffmpeg::ffmpeg_trim_and_resample::{ffmpeg_trim_and_resample, Args};
use ffmpeg_utils::ffprobe::ffprobe_get_info::ffprobe_get_info;
use filesys::file_read_bytes::file_read_bytes;
@@ -114,6 +115,13 @@ static ALLOWED_MIME_TYPES : Lazy<HashSet<&'static str>> = Lazy::new(|| {
])
});
static TRANSCODE_MIME_TYPES : Lazy<HashSet<&'static str>> = Lazy::new(|| {
HashSet::from([
// Video
"video/quicktime", // .qt quicktime files
])
});
/// This endpoint is for uploading video files.
#[utoipa::path(
post,
@@ -238,7 +246,9 @@ pub async fn upload_new_video_media_file_handler(
MediaFileUploadError::BadInput("Could not determine mimetype for file".to_string())
})?;
if !ALLOWED_MIME_TYPES.contains(mimetype.as_str()) {
let needs_transcode = TRANSCODE_MIME_TYPES.contains(mimetype.as_str());
if !ALLOWED_MIME_TYPES.contains(mimetype.as_str()) && !needs_transcode {
// NB: Don't let our error message inject malicious strings
let filtered_mimetype = mimetype
.chars()
@@ -248,16 +258,74 @@ pub async fn upload_new_video_media_file_handler(
return Err(MediaFileUploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype)));
}
// ==================== ORIGINAL FILE BOOKKEEPING ==================== //
// Hold on to the user's original file (path + mimetype + extension) so we
// can upload it as a `_original.{ext}` sibling later if we end up
// transcoding and/or resampling it.
let original_file_path: PathBuf = form.file.file.path().to_path_buf();
let original_mimetype: String = mimetype.clone();
let original_filename_extension: Option<String> = form.file.file_name.as_deref()
.and_then(|filename| std::path::Path::new(filename).extension())
.and_then(|ext| ext.to_str())
.map(|ext| ext.to_string());
let original_extension: String = mimetype_to_extension(&original_mimetype)
.map(|ext| ext.to_string())
.or(original_filename_extension)
.ok_or_else(|| {
warn!("Could not determine file extension for original mimetype: {}", &original_mimetype);
MediaFileUploadError::ServerError
})?;
// ==================== OPTIONAL TRANSCODE TO MP4 ==================== //
// Some accepted source mime types (e.g. QuickTime .mov) are not the canonical
// distribution format we want for the media file record. Transcode them to
// mp4 in a tempdir so we can resample / upload mp4 instead.
// Tempdirs we need to keep alive across the upload step. They get dropped at
// function exit which cleans up the on-disk files.
let mut transcode_tempdir_ref = None;
let mut resample_tempdir_ref = None;
let mut final_upload_file_path = form.file.file.path().to_path_buf();
if needs_transcode {
let transcode_tempdir = server_state.temp_dir_creator.new_tempdir("ffmpeg_transcode")
.map_err(|err| {
error!("Problem creating transcode temp dir: {:?}", err);
MediaFileUploadError::ServerError
})?;
let transcoded_path = transcode_tempdir.path().join("transcoded.mp4");
info!("Transcoding {} → mp4", &original_mimetype);
ffmpeg_transcode_to_mp4(FfmpegTranscodeToMp4Args {
video_input_path: form.file.file.path(),
video_output_path: &transcoded_path,
}).map_err(|err| {
error!("Problem transcoding video to mp4: {:?}", err);
MediaFileUploadError::ServerError
})?;
// The canonical file is now the transcoded mp4. Read its bytes and pin
// its mimetype to mp4 so all the downstream metadata reflects the
// transcoded file rather than the user's original upload.
file_bytes = file_read_bytes(&transcoded_path)
.map_err(|e| {
error!("Problem reading transcoded mp4: {:?}", e);
MediaFileUploadError::ServerError
})?;
mimetype = "video/mp4".to_string();
final_upload_file_path = transcoded_path;
transcode_tempdir_ref = Some(transcode_tempdir);
}
// ==================== OPTIONAL VIDEO RESAMPLE ==================== //
let should_resample = form.maybe_resample_fps.is_some()
|| form.maybe_trim_start_millis.is_some()
|| form.maybe_trim_end_millis.is_some();
let mut save_tempdir_ref = None; // NB: Save from Drop
let mut final_upload_file_path = form.file.file.path().to_path_buf();
if should_resample {
// TODO(bt,2024-09-11): Should include entropy so concurrent requests don't overwrite
let frame_temp_dir = server_state.temp_dir_creator.new_tempdir("ffmpeg")
@@ -272,8 +340,10 @@ pub async fn upload_new_video_media_file_handler(
let maybe_start_offset = form.maybe_trim_start_millis.map(|millis| Duration::from_millis(millis.0));
let maybe_end_offset = form.maybe_trim_end_millis.map(|millis| Duration::from_millis(millis.0));
// NB: Resample reads from `final_upload_file_path` (which is the transcoded
// mp4 if we transcoded above, or the user's original upload otherwise).
ffmpeg_trim_and_resample(Args {
video_input_path: form.file.file.path(),
video_input_path: &final_upload_file_path,
video_output_path: &video_output_path,
maybe_new_frame_rate,
maybe_start_offset,
@@ -297,9 +367,13 @@ pub async fn upload_new_video_media_file_handler(
})?;
final_upload_file_path = video_output_path;
save_tempdir_ref = Some(frame_temp_dir); // NB: Keep from going out of scope
resample_tempdir_ref = Some(frame_temp_dir); // NB: Keep from going out of scope
}
// True if we touched the user's file in any way (transcoded, resampled, or both).
// We'll upload their unmodified original alongside the canonical file when so.
let should_upload_original = needs_transcode || should_resample;
// ==================== OTHER FILE METADATA ==================== //
let mut maybe_duration_millis = None;
@@ -358,6 +432,47 @@ pub async fn upload_new_video_media_file_handler(
MediaFileUploadError::ServerError
})?;
// ==================== UPLOAD ORIGINAL SIBLING ==================== //
// If we transcoded and/or resampled the user's upload, also upload the
// unmodified original alongside the canonical file as `_original.{ext}`
// (sharing the same object hash / directory). The media file record points
// at the canonical file, not this sibling.
if should_upload_original {
// Build a sibling path that lives in the same directory as the canonical
// file (same hash) but with `_original.{ext}` baked into the suffix.
let original_suffix = format!("_original.{}", original_extension);
let original_bucket_path = MediaFileBucketPath::from_object_hash(
public_upload_path.get_object_hash(),
PREFIX,
Some(&original_suffix),
);
info!(
"Uploading original sibling to bucket path: {}",
original_bucket_path.get_full_object_path_str(),
);
let original_bytes = file_read_bytes(&original_file_path)
.map_err(|e| {
error!("Problem reading original file for sibling upload: {:?}", e);
MediaFileUploadError::ServerError
})?;
server_state.public_bucket_client.upload_file_with_content_type(
original_bucket_path.get_full_object_path_str(),
original_bytes.as_ref(),
&original_mimetype,
)
.await
.map_err(|e| {
// NB: Sibling failure is non-fatal — the canonical upload already
// succeeded and the user can still use the media. Log loudly.
warn!("Upload original sibling to bucket error: {:?}", e);
})
.ok();
}
let maybe_scene_source_media_file_token = form.maybe_scene_source_media_file_token
.as_ref()
.map(|token| &token.0);
@@ -4,8 +4,8 @@ import Media from "../pages/media";
import PressKit from "../pages/press-kit";
import Navbar from "../components/navbar";
import { ToastContainer } from "../components/toast/toast";
import CreateImage from "../pages/create-image";
import CreateVideo from "../pages/create-video";
// import CreateImage from "../pages/create-image";
// import CreateVideo from "../pages/create-video";
import Landing2 from "../pages/landing2";
import LandingSD2 from "../pages/landing-sd2";
import TutorialsPage from "../pages/tutorials";
@@ -30,8 +30,8 @@ export function App() {
<Routes>
<Route path="/" element={<Landing2 />} />
<Route path="/create-image" element={<CreateImage />} />
<Route path="/create-video" element={<CreateVideo />} />
{/* <Route path="/create-image" element={<CreateImage />} />
<Route path="/create-video" element={<CreateVideo />} /> */}
<Route path="/seedance-2" element={<LandingSD2 />} />
<Route path="/download" element={<Download />} />
<Route path="/media" element={<Media />} />
@@ -50,7 +50,9 @@ const ROUTE_TO_FILTER: Record<string, string> = {
meshes: "meshes",
};
const getFilterMediaClass = (filter: string): FilterMediaClasses[] | undefined => {
const getFilterMediaClass = (
filter: string,
): FilterMediaClasses[] | undefined => {
switch (filter) {
case "image":
return [FilterMediaClasses.IMAGE];
@@ -59,7 +61,11 @@ const getFilterMediaClass = (filter: string): FilterMediaClasses[] | undefined =
case "meshes":
return [FilterMediaClasses.DIMENSIONAL];
default:
return [FilterMediaClasses.IMAGE, FilterMediaClasses.VIDEO, FilterMediaClasses.DIMENSIONAL];
return [
FilterMediaClasses.IMAGE,
FilterMediaClasses.VIDEO,
FilterMediaClasses.DIMENSIONAL,
];
}
};
@@ -91,7 +97,9 @@ const getLabel = (item: any) => {
export default function Library() {
const { filter: filterParam } = useParams<{ filter?: string }>();
const navigate = useNavigate();
const activeFilter = filterParam ? (ROUTE_TO_FILTER[filterParam] ?? "all") : "all";
const activeFilter = filterParam
? (ROUTE_TO_FILTER[filterParam] ?? "all")
: "all";
const [username, setUsername] = useState<string | null>(null);
const [isLoggedIn, setIsLoggedIn] = useState<boolean | null>(null);
@@ -113,11 +121,7 @@ export default function Library() {
const checkSession = async () => {
const usersApi = new UsersApi();
const response = await usersApi.GetSession();
if (
response.success &&
response.data?.loggedIn &&
response.data.user
) {
if (response.success && response.data?.loggedIn && response.data.user) {
setUsername(response.data.user.username);
setIsLoggedIn(true);
} else {
@@ -135,8 +139,8 @@ export default function Library() {
const thumbnail = isDimensional
? null
: getMediaThumbnail(item.media_links, item.media_class, {
size: THUMBNAIL_SIZES.MEDIUM,
});
size: THUMBNAIL_SIZES.MEDIUM,
});
return {
id: item.token,
@@ -169,7 +173,9 @@ export default function Library() {
if (response.success && response.data) {
const newItems = response.data
.filter((item: any) => item.media_type !== FilterMediaType.SCENE_JSON)
.filter(
(item: any) => item.media_type !== FilterMediaType.SCENE_JSON,
)
.map(mapApiItem);
if (reset) {
@@ -250,17 +256,17 @@ export default function Library() {
const navigatePrev =
currentIndex > 0
? () => {
const prev = flatItems[currentIndex - 1];
setLightboxItem(prev);
}
const prev = flatItems[currentIndex - 1];
setLightboxItem(prev);
}
: undefined;
const navigateNext =
currentIndex >= 0 && currentIndex < flatItems.length - 1
? () => {
const next = flatItems[currentIndex + 1];
setLightboxItem(next);
}
const next = flatItems[currentIndex + 1];
setLightboxItem(next);
}
: undefined;
const handleItemDeleted = useCallback((id: string) => {
@@ -341,10 +347,11 @@ export default function Library() {
<button
key={filter.id}
onClick={() => navigate(filter.route)}
className={`flex items-center gap-2 px-4 py-1.5 rounded-md text-sm font-medium transition-colors ${activeFilter === filter.id
className={`flex items-center gap-2 px-4 py-1.5 rounded-md text-sm font-medium transition-colors ${
activeFilter === filter.id
? "bg-ui-controls text-white"
: "text-white/60 hover:text-white"
}`}
}`}
>
<FontAwesomeIcon icon={filter.icon} className="text-xs" />
{filter.label}
@@ -386,10 +393,8 @@ export default function Library() {
</div>
) : allItems.length === 0 && !loading ? (
<div className="flex flex-col items-center justify-center py-20">
<p className="text-white/40 text-sm mb-4">
No items yet. Start generating!
</p>
<div className="flex gap-3">
<p className="text-white/40 text-sm mb-4">No items yet.</p>
{/* <div className="flex gap-3">
<Link to="/create-image">
<Button variant="primary" className="text-sm px-4 py-2">
Create Image
@@ -403,7 +408,7 @@ export default function Library() {
Create Video
</Button>
</Link>
</div>
</div> */}
</div>
) : (
<div className="space-y-6">
+38 -4
View File
@@ -4,6 +4,40 @@ import { API_TARGETS } from "./enums/Api.js";
//import { fetch } from '@tauri-apps/plugin-http'
import { FetchProxy as fetch } from "@storyteller/tauri-utils";
const SESSION_STORAGE_KEY = "artcraft_signed_session";
/** Store a signed session JWT for use as a header fallback (mobile browsers block 3rd-party cookies). */
export function storeSignedSession(signedSession: string) {
try {
localStorage.setItem(SESSION_STORAGE_KEY, signedSession);
} catch { /* localStorage unavailable */ }
}
/** Clear stored session (on logout). */
export function clearSignedSession() {
try {
localStorage.removeItem(SESSION_STORAGE_KEY);
} catch { /* localStorage unavailable */ }
}
/** Get the stored session token, if any. */
export function getSignedSession(): string | null {
try {
return localStorage.getItem(SESSION_STORAGE_KEY);
} catch {
return null;
}
}
/** Build headers with session fallback for mobile browsers. */
export function buildSessionHeaders(base: Record<string, string>): Record<string, string> {
const session = getSignedSession();
if (session) {
return { ...base, session };
}
return base;
}
type NonNullableObject<T extends object> = NonNullable<T>;
export interface ApiResponse<T, P = undefined> {
@@ -62,10 +96,10 @@ export class ApiManager {
const response = await fetch(endpointWithQueries, {
method,
headers: {
headers: buildSessionHeaders({
Accept: "application/json",
"Content-Type": "application/json",
},
}),
credentials: "include",
body: bodyInString,
});
@@ -89,9 +123,9 @@ export class ApiManager {
): Promise<T> {
const response = await fetch(endpoint, {
method,
headers: {
headers: buildSessionHeaders({
Accept: "application/json",
},
}),
credentials: "include",
body: body,
});
+3 -3
View File
@@ -1,4 +1,4 @@
import { ApiManager, ApiResponse } from "./ApiManager";
import { ApiManager, ApiResponse, buildSessionHeaders } from "./ApiManager";
export class EngineApi extends ApiManager {
public async ConvertTbxToGltf({
@@ -57,9 +57,9 @@ export class EngineApi extends ApiManager {
const response = await fetch(endpoint, {
method: "POST",
headers: {
headers: buildSessionHeaders({
Accept: "application/json",
},
}),
credentials: "include",
body: formData,
});
@@ -1,4 +1,4 @@
import { ApiManager, ApiResponse } from "./ApiManager.js";
import { ApiManager, ApiResponse, buildSessionHeaders } from "./ApiManager.js";
import { FetchProxy as fetch } from "@storyteller/tauri-utils";
export class PasswordResetApi extends ApiManager {
@@ -61,10 +61,10 @@ export class PasswordResetApi extends ApiManager {
try {
const response = await fetch(endpoint, {
method: "POST",
headers: {
headers: buildSessionHeaders({
Accept: "application/json",
"Content-Type": "application/json",
},
}),
credentials: "include",
body: JSON.stringify(body),
});
+3 -3
View File
@@ -1,4 +1,4 @@
import { ApiManager, ApiResponse } from "./ApiManager.js";
import { ApiManager, ApiResponse, buildSessionHeaders } from "./ApiManager.js";
import { Prompts } from "./models/Prompts.js";
import { FetchProxy as fetch } from "@storyteller/tauri-utils";
@@ -209,9 +209,9 @@ export class PromptsApi extends ApiManager {
const response = await fetch(endpoint, {
method: "POST",
headers: {
headers: buildSessionHeaders({
Accept: "application/json",
},
}),
credentials: "include",
body: formData,
});
+10 -3
View File
@@ -1,4 +1,4 @@
import { ApiManager, ApiResponse } from "./ApiManager.js";
import { ApiManager, ApiResponse, buildSessionHeaders, storeSignedSession, clearSignedSession } from "./ApiManager.js";
import { UserInfo } from "./models/Users.js";
import { FetchProxy as fetch } from "@storyteller/tauri-utils";
@@ -27,10 +27,10 @@ export class UsersApi extends ApiManager {
const response = await fetch(endpoint, {
method,
headers: {
headers: buildSessionHeaders({
Accept: "application/json",
"Content-Type": "application/json",
},
}),
credentials: "include",
body: bodyInString,
});
@@ -141,6 +141,9 @@ export class UsersApi extends ApiManager {
method: "POST",
body: body,
});
if (response.success && response.signed_session) {
storeSignedSession(response.signed_session);
}
return {
success: response.success,
data: response.success
@@ -157,6 +160,7 @@ export class UsersApi extends ApiManager {
}
public Logout(): Promise<ApiResponse<null>> {
clearSignedSession();
const endpoint = `${this.getApiSchemeAndHost()}/v1/logout`;
return this.post<null, { success: boolean; error_message?: string }>({
endpoint: endpoint,
@@ -245,6 +249,9 @@ export class UsersApi extends ApiManager {
body: body,
});
if (response.success && response.signed_session) {
storeSignedSession(response.signed_session);
}
return {
success: response.success,
data: response.success
@@ -137,7 +137,7 @@ export const GalleryDraggableItem: React.FC<GalleryDraggableItemProps> = ({
tabIndex={-1}
className={twMerge(
"w-full group relative overflow-visible rounded-md border-[3px] transition-all focus:outline-none",
"aspect-square",
item.mediaClass === "video" ? "aspect-video" : "aspect-square",
selected || bulkSelected
? "border-primary"
: disableTooltipAndBadge
@@ -171,7 +171,7 @@ export const GalleryDraggableItem: React.FC<GalleryDraggableItemProps> = ({
alt={item.label}
className={twMerge(
"h-full w-full bg-black/30",
imageFit === "contain" ? "object-contain" : "object-cover",
item.mediaClass === "video" || imageFit === "contain" ? "object-contain" : "object-cover",
)}
draggable={false}
onError={onImageError}
@@ -187,7 +187,7 @@ export const GalleryDraggableItem: React.FC<GalleryDraggableItemProps> = ({
);
return (
<div className={twMerge("group relative w-full", "aspect-square")}>
<div className={twMerge("group relative w-full", item.mediaClass === "video" ? "aspect-video" : "aspect-square")}>
{/* dropdown menu */}
{mode !== "select" && (
<div