Compare commits

...
Author SHA1 Message Date
Brandon Thomas 932431147f client fixes 2025-12-22 00:35:24 -05:00
Brandon Thomas 49f6913391 clients 2025-12-21 23:54:24 -05:00
Brandon Thomas 71645335d6 veo 3 client 2025-12-21 23:44:35 -05:00
Brandon Thomas 36e5a319f5 sora 2 pro 2025-12-21 23:33:04 -05:00
Brandon Thomas 9130bd1ddd sora2 2025-12-21 23:23:50 -05:00
Brandon Thomas d286f73d7f wip 2025-12-21 23:06:03 -05:00
Brandon Thomas 03c0f6b640 veo 3.1 fast 2025-12-21 22:49:00 -05:00
Brandon Thomas fd141a5c36 veo3.1 2025-12-21 22:38:43 -05:00
Brandon Thomas de105e1cc4 kling 2.6 handler 2025-12-21 20:38:54 -05:00
Brandon Thomas 330184838e fix kling handler 2025-12-21 20:20:40 -05:00
Brandon Thomas bf758cb10d kling 2.5 turbo handler 2025-12-21 20:14:17 -05:00
Brandon Thomas f512f3cb7d sora2 (non-pro) 2025-12-21 20:14:17 -05:00
Brandon Thomas b80872647d sora 2 pro 2025-12-21 20:14:16 -05:00
Brandon Thomas 6a692b9d2c veo3.1 fast 2025-12-21 20:14:15 -05:00
Brandon Thomas 10f0895f2a defs for veo 3.1 2025-12-21 20:14:14 -05:00
Brandon Thomas 476a26181f kling 2.6 2025-12-21 20:14:13 -05:00
Brandon Thomas bf0a563efb kling 2.5 defs 2025-12-21 20:14:12 -05:00
32 changed files with 2742 additions and 15 deletions
@@ -5,4 +5,5 @@ pub mod generate_seedance_1_0_lite_image_to_video;
pub mod generate_seedance_1_0_pro_image_to_video;
pub mod generate_veo_2_image_to_video;
pub mod generate_veo_3_fast_image_to_video;
pub mod generate_veo_3_image_to_video;
pub mod generate_veo_3_image_to_video;
pub mod multi_function;
@@ -0,0 +1,64 @@
use serde::{Deserialize, Serialize};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
pub const KLING_2P5_TURBO_PRO_MULTI_FUNCTION_VIDEO_VIDEO_PATH: &str = "/v1/generate/video/multi_function/kling_2p5_turbo_pro";
/// Both text-to-video and image-to-video in one request.
#[derive(Serialize, Deserialize, ToSchema)]
pub struct Kling2p5TurboProMultiFunctionVideoGenRequest {
/// Idempotency token to prevent duplicate requests.
pub uuid_idempotency_token: String,
/// "Required".
/// Required downstream, but we'll coerce null to empty string.
/// Text prompt.
pub prompt: Option<String>,
/// Optional.
/// Only for text-to-video.
pub negative_prompt: Option<String>,
/// Optional.
/// Only for image-to-video
/// Source of the starting frame
/// If present, we're doing image-to-video
/// If absent, we're doing text-to-video
pub start_frame_image_media_token: Option<MediaFileToken>,
/// Optional.
/// Only for image-to-video
/// Source of the ending frame (if desired)
pub end_frame_image_media_token: Option<MediaFileToken>,
/// Optional
/// Duration of the video
pub duration: Option<Kling2p5TurboProMultiFunctionVideoGenDuration>,
/// Optional.
/// Only for text-to-video
pub aspect_ratio: Option<Kling2p5TurboProMultiFunctionVideoGenAspectRatio>,
}
#[derive(Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Kling2p5TurboProMultiFunctionVideoGenDuration {
FiveSeconds,
TenSeconds,
}
#[derive(Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Kling2p5TurboProMultiFunctionVideoGenAspectRatio {
Square,
SixteenByNine,
NineBySixteen,
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct Kling2p5TurboProMultiFunctionVideoGenResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
@@ -0,0 +1,68 @@
use serde::{Deserialize, Serialize};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
pub const KLING_2P6_PRO_MULTI_FUNCTION_VIDEO_VIDEO_PATH: &str = "/v1/generate/video/multi_function/kling_2p6_pro";
/// Both text-to-video and image-to-video in one request.
#[derive(Serialize, Deserialize, ToSchema)]
pub struct Kling2p6ProMultiFunctionVideoGenRequest {
/// Idempotency token to prevent duplicate requests.
pub uuid_idempotency_token: String,
/// "Required".
/// Required downstream, but we'll coerce null to empty string.
/// Text prompt.
pub prompt: Option<String>,
/// Optional.
/// Only for text-to-video.
pub negative_prompt: Option<String>,
/// Optional.
/// Only for image-to-video
/// Source of the starting frame
/// If present, we're doing image-to-video
/// If absent, we're doing text-to-video
pub start_frame_image_media_token: Option<MediaFileToken>,
// /// Optional.
// /// Only for image-to-video
// /// Source of the ending frame (if desired)
// pub end_frame_image_media_token: Option<MediaFileToken>,
/// Optional
/// Duration of the video
pub duration: Option<Kling2p6ProMultiFunctionVideoGenDuration>,
/// Optional
/// Whether to generate audio.
pub generate_audio: Option<bool>,
/// Optional.
/// Only for text-to-video
pub aspect_ratio: Option<Kling2p6ProMultiFunctionVideoGenAspectRatio>,
}
#[derive(Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Kling2p6ProMultiFunctionVideoGenDuration {
FiveSeconds,
TenSeconds,
}
#[derive(Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Kling2p6ProMultiFunctionVideoGenAspectRatio {
Square,
SixteenByNine,
NineBySixteen,
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct Kling2p6ProMultiFunctionVideoGenResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
@@ -0,0 +1,6 @@
pub mod kling_2_5_turbo_multi_function_video_gen;
pub mod kling_2_6_multi_function_video_gen;
pub mod sora_2_multi_function_video_gen;
pub mod sora_2_pro_multi_function_video_gen;
pub mod veo_3p1_fast_multi_function_video_gen;
pub mod veo_3p1_multi_function_video_gen;
@@ -0,0 +1,66 @@
use serde::{Deserialize, Serialize};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
pub const SORA_2_MULTI_FUNCTION_VIDEO_VIDEO_PATH: &str = "/v1/generate/video/multi_function/sora_2";
/// Both text-to-video and image-to-video in one request.
#[derive(Serialize, Deserialize, ToSchema)]
pub struct Sora2MultiFunctionVideoGenRequest {
/// Idempotency token to prevent duplicate requests.
pub uuid_idempotency_token: String,
/// "Required".
/// Required downstream, but we'll coerce null to empty string.
/// Text prompt.
pub prompt: Option<String>,
/// Optional.
/// Only for image-to-video
/// Source of the starting frame
/// If present, we're doing image-to-video
/// If absent, we're doing text-to-video
pub image_media_token: Option<MediaFileToken>,
/// Optional.
pub resolution: Option<Sora2MultiFunctionVideoGenResolution>,
/// Optional
/// Duration of the video
/// (this is uniform against all modes)
pub duration: Option<Sora2MultiFunctionVideoGenDuration>,
/// Optional.
pub aspect_ratio: Option<Sora2MultiFunctionVideoGenAspectRatio>,
}
#[derive(Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Sora2MultiFunctionVideoGenResolution {
Auto,
SevenTwentyP,
}
#[derive(Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Sora2MultiFunctionVideoGenDuration {
FourSeconds,
EightSeconds,
TwelveSeconds,
}
#[derive(Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Sora2MultiFunctionVideoGenAspectRatio {
Auto,
NineBySixteen,
SixteenByNine,
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct Sora2MultiFunctionVideoGenResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
@@ -0,0 +1,67 @@
use serde::{Deserialize, Serialize};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
pub const SORA_2_PRO_MULTI_FUNCTION_VIDEO_VIDEO_PATH: &str = "/v1/generate/video/multi_function/sora_2_pro";
/// Both text-to-video and image-to-video in one request.
#[derive(Serialize, Deserialize, ToSchema)]
pub struct Sora2ProMultiFunctionVideoGenRequest {
/// Idempotency token to prevent duplicate requests.
pub uuid_idempotency_token: String,
/// "Required".
/// Required downstream, but we'll coerce null to empty string.
/// Text prompt.
pub prompt: Option<String>,
/// Optional.
/// Only for image-to-video
/// Source of the starting frame
/// If present, we're doing image-to-video
/// If absent, we're doing text-to-video
pub image_media_token: Option<MediaFileToken>,
/// Optional.
pub resolution: Option<Sora2ProMultiFunctionVideoGenResolution>,
/// Optional
/// Duration of the video
/// (this is uniform against all modes)
pub duration: Option<Sora2ProMultiFunctionVideoGenDuration>,
/// Optional.
pub aspect_ratio: Option<Sora2ProMultiFunctionVideoGenAspectRatio>,
}
#[derive(Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Sora2ProMultiFunctionVideoGenResolution {
Auto,
SevenTwentyP,
TenEightyP,
}
#[derive(Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Sora2ProMultiFunctionVideoGenDuration {
FourSeconds,
EightSeconds,
TwelveSeconds,
}
#[derive(Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Sora2ProMultiFunctionVideoGenAspectRatio {
Auto,
NineBySixteen,
SixteenByNine,
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct Sora2ProMultiFunctionVideoGenResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
@@ -0,0 +1,90 @@
use serde::{Deserialize, Serialize};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
pub const VEO_3P1_FAST_MULTI_FUNCTION_VIDEO_VIDEO_PATH: &str = "/v1/generate/video/multi_function/veo_3p1_fast";
/// Both text-to-video and image-to-video in one request.
#[derive(Serialize, Deserialize, ToSchema)]
pub struct Veo3p1FastMultiFunctionVideoGenRequest {
/// Idempotency token to prevent duplicate requests.
pub uuid_idempotency_token: String,
/// "Required".
/// Required downstream, but we'll coerce null to empty string.
/// Text prompt.
pub prompt: Option<String>,
/// Optional.
/// Only for text-to-video.
pub negative_prompt: Option<String>,
/// Optional.
/// Only for image-to-video
/// Source of the starting frame
/// If present, we're doing image-to-video
/// If absent, we're doing text-to-video
pub start_frame_image_media_token: Option<MediaFileToken>,
/// Optional.
/// Only for image-to-video
/// Source of the ending frame (if desired)
pub end_frame_image_media_token: Option<MediaFileToken>,
/// Optional
/// Duration of the video
/// (this is uniform against all modes)
pub duration: Option<Veo3p1FastMultiFunctionVideoGenDuration>,
/// Optional.
pub aspect_ratio: Option<Veo3p1FastMultiFunctionVideoGenAspectRatio>,
/// Optional.
pub resolution: Option<Veo3p1FastMultiFunctionVideoGenResolution>,
/// Optional
/// Whether to generate audio.
pub generate_audio: Option<bool>,
/// Optional
/// Only for text-to-video
pub enhance_prompt: Option<bool>,
/// Optional
/// Only for text-to-video
pub seed: Option<i64>,
/// Optional
/// Only for text-to-video
pub auto_fix: Option<bool>,
}
#[derive(Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Veo3p1FastMultiFunctionVideoGenDuration {
FourSeconds,
SixSeconds,
EightSeconds,
}
#[derive(Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Veo3p1FastMultiFunctionVideoGenAspectRatio {
Auto,
SixteenByNine,
NineBySixteen,
}
#[derive(Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Veo3p1FastMultiFunctionVideoGenResolution {
SevenTwentyP,
TenEightyP,
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct Veo3p1FastMultiFunctionVideoGenResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
@@ -0,0 +1,90 @@
use serde::{Deserialize, Serialize};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
pub const VEO_3P1_MULTI_FUNCTION_VIDEO_VIDEO_PATH: &str = "/v1/generate/video/multi_function/veo_3p1";
/// Both text-to-video and image-to-video in one request.
#[derive(Serialize, Deserialize, ToSchema)]
pub struct Veo3p1MultiFunctionVideoGenRequest {
/// Idempotency token to prevent duplicate requests.
pub uuid_idempotency_token: String,
/// "Required".
/// Required downstream, but we'll coerce null to empty string.
/// Text prompt.
pub prompt: Option<String>,
/// Optional.
/// Only for text-to-video.
pub negative_prompt: Option<String>,
/// Optional.
/// Only for image-to-video
/// Source of the starting frame
/// If present, we're doing image-to-video
/// If absent, we're doing text-to-video
pub start_frame_image_media_token: Option<MediaFileToken>,
/// Optional.
/// Only for image-to-video
/// Source of the ending frame (if desired)
pub end_frame_image_media_token: Option<MediaFileToken>,
/// Optional
/// Duration of the video
/// (this is uniform against all modes)
pub duration: Option<Veo3p1MultiFunctionVideoGenDuration>,
/// Optional.
pub aspect_ratio: Option<Veo3p1MultiFunctionVideoGenAspectRatio>,
/// Optional.
pub resolution: Option<Veo3p1MultiFunctionVideoGenResolution>,
/// Optional
/// Whether to generate audio.
pub generate_audio: Option<bool>,
/// Optional
/// Only for text-to-video
pub enhance_prompt: Option<bool>,
/// Optional
/// Only for text-to-video
pub seed: Option<i64>,
/// Optional
/// Only for text-to-video
pub auto_fix: Option<bool>,
}
#[derive(Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Veo3p1MultiFunctionVideoGenDuration {
FourSeconds,
SixSeconds,
EightSeconds,
}
#[derive(Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Veo3p1MultiFunctionVideoGenAspectRatio {
Auto,
SixteenByNine,
NineBySixteen,
}
#[derive(Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Veo3p1MultiFunctionVideoGenResolution {
SevenTwentyP,
TenEightyP,
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct Veo3p1MultiFunctionVideoGenResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
@@ -5,3 +5,4 @@ pub mod generate_seedance_1_0_lite_image_to_video;
pub mod generate_veo_2_image_to_video;
pub mod generate_veo_3_fast_image_to_video;
pub mod generate_veo_3_image_to_video;
pub mod multi_function;
@@ -0,0 +1,18 @@
use crate::credentials::storyteller_credential_set::StorytellerCredentialSet;
use crate::error::storyteller_error::StorytellerError;
use crate::utils::api_host::ApiHost;
use crate::utils::basic_json_post_request::basic_json_post_request;
use artcraft_api_defs::generate::video::multi_function::kling_2_5_turbo_multi_function_video_gen::{Kling2p5TurboProMultiFunctionVideoGenRequest, Kling2p5TurboProMultiFunctionVideoGenResponse, KLING_2P5_TURBO_PRO_MULTI_FUNCTION_VIDEO_VIDEO_PATH};
pub async fn kling_2p5_turbo_pro_multi_function_image_gen(
api_host: &ApiHost,
maybe_creds: Option<&StorytellerCredentialSet>,
request: Kling2p5TurboProMultiFunctionVideoGenRequest,
) -> Result<Kling2p5TurboProMultiFunctionVideoGenResponse, StorytellerError> {
Ok(basic_json_post_request(
api_host,
KLING_2P5_TURBO_PRO_MULTI_FUNCTION_VIDEO_VIDEO_PATH,
maybe_creds,
request,
).await?)
}
@@ -0,0 +1,18 @@
use crate::credentials::storyteller_credential_set::StorytellerCredentialSet;
use crate::error::storyteller_error::StorytellerError;
use crate::utils::api_host::ApiHost;
use crate::utils::basic_json_post_request::basic_json_post_request;
use artcraft_api_defs::generate::video::multi_function::kling_2_6_multi_function_video_gen::{Kling2p6ProMultiFunctionVideoGenRequest, Kling2p6ProMultiFunctionVideoGenResponse, KLING_2P6_PRO_MULTI_FUNCTION_VIDEO_VIDEO_PATH};
pub async fn kling_2p6_pro_multi_function_image_gen(
api_host: &ApiHost,
maybe_creds: Option<&StorytellerCredentialSet>,
request: Kling2p6ProMultiFunctionVideoGenRequest,
) -> Result<Kling2p6ProMultiFunctionVideoGenResponse, StorytellerError> {
Ok(basic_json_post_request(
api_host,
KLING_2P6_PRO_MULTI_FUNCTION_VIDEO_VIDEO_PATH,
maybe_creds,
request,
).await?)
}
@@ -0,0 +1,6 @@
pub mod kling_2p5_turbo_pro_multi_function_video_gen;
pub mod kling_2p6_pro_multi_function_video_gen;
pub mod sora_2_multi_function_video_gen;
pub mod sora_2_pro_multi_function_video_gen;
pub mod veo_3p1_fast_multi_function_video_gen;
pub mod veo_3p1_multi_function_video_gen;
@@ -0,0 +1,18 @@
use crate::credentials::storyteller_credential_set::StorytellerCredentialSet;
use crate::error::storyteller_error::StorytellerError;
use crate::utils::api_host::ApiHost;
use crate::utils::basic_json_post_request::basic_json_post_request;
use artcraft_api_defs::generate::video::multi_function::sora_2_multi_function_video_gen::{Sora2MultiFunctionVideoGenRequest, Sora2MultiFunctionVideoGenResponse, SORA_2_MULTI_FUNCTION_VIDEO_VIDEO_PATH};
pub async fn sora_2_multi_function_image_gen(
api_host: &ApiHost,
maybe_creds: Option<&StorytellerCredentialSet>,
request: Sora2MultiFunctionVideoGenRequest,
) -> Result<Sora2MultiFunctionVideoGenResponse, StorytellerError> {
Ok(basic_json_post_request(
api_host,
SORA_2_MULTI_FUNCTION_VIDEO_VIDEO_PATH,
maybe_creds,
request,
).await?)
}
@@ -0,0 +1,18 @@
use crate::credentials::storyteller_credential_set::StorytellerCredentialSet;
use crate::error::storyteller_error::StorytellerError;
use crate::utils::api_host::ApiHost;
use crate::utils::basic_json_post_request::basic_json_post_request;
use artcraft_api_defs::generate::video::multi_function::sora_2_pro_multi_function_video_gen::{Sora2ProMultiFunctionVideoGenRequest, Sora2ProMultiFunctionVideoGenResponse, SORA_2_PRO_MULTI_FUNCTION_VIDEO_VIDEO_PATH};
pub async fn sora_2_fast_multi_function_image_gen(
api_host: &ApiHost,
maybe_creds: Option<&StorytellerCredentialSet>,
request: Sora2ProMultiFunctionVideoGenRequest,
) -> Result<Sora2ProMultiFunctionVideoGenResponse, StorytellerError> {
Ok(basic_json_post_request(
api_host,
SORA_2_PRO_MULTI_FUNCTION_VIDEO_VIDEO_PATH,
maybe_creds,
request,
).await?)
}
@@ -0,0 +1,18 @@
use crate::credentials::storyteller_credential_set::StorytellerCredentialSet;
use crate::error::storyteller_error::StorytellerError;
use crate::utils::api_host::ApiHost;
use crate::utils::basic_json_post_request::basic_json_post_request;
use artcraft_api_defs::generate::video::multi_function::veo_3p1_fast_multi_function_video_gen::{Veo3p1FastMultiFunctionVideoGenRequest, Veo3p1FastMultiFunctionVideoGenResponse, VEO_3P1_FAST_MULTI_FUNCTION_VIDEO_VIDEO_PATH};
pub async fn veo_3p1_fast_multi_function_image_gen(
api_host: &ApiHost,
maybe_creds: Option<&StorytellerCredentialSet>,
request: Veo3p1FastMultiFunctionVideoGenRequest,
) -> Result<Veo3p1FastMultiFunctionVideoGenResponse, StorytellerError> {
Ok(basic_json_post_request(
api_host,
VEO_3P1_FAST_MULTI_FUNCTION_VIDEO_VIDEO_PATH,
maybe_creds,
request,
).await?)
}
@@ -0,0 +1,18 @@
use crate::credentials::storyteller_credential_set::StorytellerCredentialSet;
use crate::error::storyteller_error::StorytellerError;
use crate::utils::api_host::ApiHost;
use crate::utils::basic_json_post_request::basic_json_post_request;
use artcraft_api_defs::generate::video::multi_function::veo_3p1_multi_function_video_gen::{Veo3p1MultiFunctionVideoGenRequest, Veo3p1MultiFunctionVideoGenResponse, VEO_3P1_MULTI_FUNCTION_VIDEO_VIDEO_PATH};
pub async fn veo_3p1_multi_function_image_gen(
api_host: &ApiHost,
maybe_creds: Option<&StorytellerCredentialSet>,
request: Veo3p1MultiFunctionVideoGenRequest,
) -> Result<Veo3p1MultiFunctionVideoGenResponse, StorytellerError> {
Ok(basic_json_post_request(
api_host,
VEO_3P1_MULTI_FUNCTION_VIDEO_VIDEO_PATH,
maybe_creds,
request,
).await?)
}
@@ -36,7 +36,6 @@ pub enum EnqueueVeo3p1FastFirstLastFrameImageToVideoDurationSeconds {
#[derive(Copy, Clone, Debug)]
pub enum EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio {
Auto,
Square,
SixteenByNine,
NineBySixteen,
}
@@ -62,7 +61,6 @@ pub async fn enqueue_veo_3p1_fast_first_last_frame_image_to_video_webhook<R: Int
let aspect_ratio = args.aspect_ratio
.map(|aspect_ratio| match aspect_ratio {
EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio::Auto => "auto",
EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio::Square => "1:1",
EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio::SixteenByNine => "16:9",
EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio::NineBySixteen => "9:16",
})
@@ -33,7 +33,6 @@ pub enum EnqueueVeo3p1FastImageToVideoDurationSeconds {
#[derive(Copy, Clone, Debug)]
pub enum EnqueueVeo3p1FastImageToVideoAspectRatio {
Auto,
Square,
SixteenByNine,
NineBySixteen,
}
@@ -59,7 +58,6 @@ pub async fn enqueue_veo_3p1_fast_image_to_video_webhook<R: IntoUrl>(
let aspect_ratio = args.aspect_ratio
.map(|aspect_ratio| match aspect_ratio {
EnqueueVeo3p1FastImageToVideoAspectRatio::Auto => "auto",
EnqueueVeo3p1FastImageToVideoAspectRatio::Square => "1:1",
EnqueueVeo3p1FastImageToVideoAspectRatio::SixteenByNine => "16:9",
EnqueueVeo3p1FastImageToVideoAspectRatio::NineBySixteen => "9:16",
})
@@ -36,7 +36,6 @@ pub enum EnqueueVeo3p1FirstLastFrameImageToVideoDurationSeconds {
#[derive(Copy, Clone, Debug)]
pub enum EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio {
Auto,
Square,
SixteenByNine,
NineBySixteen,
}
@@ -62,7 +61,6 @@ pub async fn enqueue_veo_3p1_first_last_frame_image_to_video_webhook<R: IntoUrl>
let aspect_ratio = args.aspect_ratio
.map(|aspect_ratio| match aspect_ratio {
EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio::Auto => "auto",
EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio::Square => "1:1",
EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio::SixteenByNine => "16:9",
EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio::NineBySixteen => "9:16",
})
@@ -33,7 +33,6 @@ pub enum EnqueueVeo3p1ImageToVideoDurationSeconds {
#[derive(Copy, Clone, Debug)]
pub enum EnqueueVeo3p1ImageToVideoAspectRatio {
Auto,
Square,
SixteenByNine,
NineBySixteen,
}
@@ -59,7 +58,6 @@ pub async fn enqueue_veo_3p1_image_to_video_webhook<R: IntoUrl>(
let aspect_ratio = args.aspect_ratio
.map(|aspect_ratio| match aspect_ratio {
EnqueueVeo3p1ImageToVideoAspectRatio::Auto => "auto",
EnqueueVeo3p1ImageToVideoAspectRatio::Square => "1:1",
EnqueueVeo3p1ImageToVideoAspectRatio::SixteenByNine => "16:9",
EnqueueVeo3p1ImageToVideoAspectRatio::NineBySixteen => "9:16",
})
@@ -34,7 +34,6 @@ pub enum EnqueueVeo3p1FastTextToVideoDurationSeconds {
#[derive(Copy, Clone, Debug)]
pub enum EnqueueVeo3p1FastTextToVideoAspectRatio {
Auto,
Square,
SixteenByNine,
NineBySixteen,
}
@@ -60,7 +59,6 @@ pub async fn enqueue_veo_3p1_fast_text_to_video_webhook<R: IntoUrl>(
let aspect_ratio = args.aspect_ratio
.map(|aspect_ratio| match aspect_ratio {
EnqueueVeo3p1FastTextToVideoAspectRatio::Auto => "auto",
EnqueueVeo3p1FastTextToVideoAspectRatio::Square => "1:1",
EnqueueVeo3p1FastTextToVideoAspectRatio::SixteenByNine => "16:9",
EnqueueVeo3p1FastTextToVideoAspectRatio::NineBySixteen => "9:16",
})
@@ -34,7 +34,6 @@ pub enum EnqueueVeo3p1TextToVideoDurationSeconds {
#[derive(Copy, Clone, Debug)]
pub enum EnqueueVeo3p1TextToVideoAspectRatio {
Auto,
Square,
SixteenByNine,
NineBySixteen,
}
@@ -60,7 +59,6 @@ pub async fn enqueue_veo_3p1_text_to_video_webhook<R: IntoUrl>(
let aspect_ratio = args.aspect_ratio
.map(|aspect_ratio| match aspect_ratio {
EnqueueVeo3p1TextToVideoAspectRatio::Auto => "auto",
EnqueueVeo3p1TextToVideoAspectRatio::Square => "1:1",
EnqueueVeo3p1TextToVideoAspectRatio::SixteenByNine => "16:9",
EnqueueVeo3p1TextToVideoAspectRatio::NineBySixteen => "9:16",
})
@@ -85,18 +85,28 @@ pub enum ModelType {
Kling21Pro,
#[serde(rename = "kling_2p1_master")]
Kling21Master,
#[serde(rename = "kling_2p5_turbo_pro")]
Kling2p5TurboPro,
#[serde(rename = "kling_2p6_pro")]
Kling2p6Pro,
#[serde(rename = "seedance_1p0_lite")]
Seedance10Lite,
#[serde(rename = "seedance_1p0_pro")]
Seedance10Pro,
#[serde(rename = "sora_2")]
Sora2,
#[serde(rename = "sora_2_pro")]
Sora2Pro,
#[serde(rename = "veo_2")]
Veo2,
#[serde(rename = "veo_3")]
Veo3,
#[serde(rename = "veo_3_fast")]
Veo3Fast,
#[serde(rename = "veo_3p1")]
Veo3p1,
#[serde(rename = "veo_3p1_fast")]
Veo3p1Fast,
// 3D Object generation models
#[serde(rename = "hunyuan_3d_2p0")]
@@ -147,12 +157,17 @@ impl ModelType {
Self::Kling16Pro => "kling_1p6_pro",
Self::Kling21Pro => "kling_2p1_pro",
Self::Kling21Master => "kling_2p1_master",
Self::Kling2p5TurboPro => "kling_2p5_turbo_pro",
Self::Kling2p6Pro => "kling_2p6_pro",
Self::Seedance10Lite => "seedance_1p0_lite",
Self::Seedance10Pro => "seedance_1p0_pro",
Self::Sora2 => "sora_2",
Self::Sora2Pro => "sora_2_pro",
Self::Veo2 => "veo_2",
Self::Veo3 => "veo_3",
Self::Veo3Fast => "veo_3_fast",
Self::Veo3p1 => "veo_3p1",
Self::Veo3p1Fast => "veo_3p1_fast",
// 3D Object generation models
Self::Hunyuan3d2_0 => "hunyuan_3d_2p0",
@@ -195,12 +210,17 @@ impl ModelType {
"kling_1p6_pro" => Ok(Self::Kling16Pro),
"kling_2p1_pro" => Ok(Self::Kling21Pro),
"kling_2p1_master" => Ok(Self::Kling21Master),
"kling_2p5_turbo_pro" => Ok(Self::Kling2p5TurboPro),
"kling_2p6_pro" => Ok(Self::Kling2p6Pro),
"seedance_1p0_lite" => Ok(Self::Seedance10Lite),
"seedance_1p0_pro" => Ok(Self::Seedance10Pro),
"sora_2" => Ok(Self::Sora2),
"sora_2_pro" => Ok(Self::Sora2Pro),
"veo_2" => Ok(Self::Veo2),
"veo_3" => Ok(Self::Veo3),
"veo_3_fast" => Ok(Self::Veo3Fast),
"veo_3p1" => Ok(Self::Veo3p1),
"veo_3p1_fast" => Ok(Self::Veo3p1Fast),
// 3D Object generation models
"hunyuan_3d_2p0" => Ok(Self::Hunyuan3d2_0),
@@ -247,12 +267,17 @@ impl ModelType {
Self::Kling16Pro,
Self::Kling21Pro,
Self::Kling21Master,
Self::Kling2p5TurboPro,
Self::Kling2p6Pro,
Self::Seedance10Lite,
Self::Seedance10Pro,
Self::Sora2,
Self::Sora2Pro,
Self::Veo2,
Self::Veo3,
Self::Veo3Fast,
Self::Veo3p1,
Self::Veo3p1Fast,
// 3D Object generation models
Self::Hunyuan3d2_0,
@@ -303,12 +328,17 @@ mod tests {
assert_serialization(ModelType::Kling16Pro, "kling_1p6_pro");
assert_serialization(ModelType::Kling21Pro, "kling_2p1_pro");
assert_serialization(ModelType::Kling21Master, "kling_2p1_master");
assert_serialization(ModelType::Kling2p5TurboPro, "kling_2p5_turbo_pro");
assert_serialization(ModelType::Kling2p6Pro, "kling_2p6_pro");
assert_serialization(ModelType::Seedance10Lite, "seedance_1p0_lite");
assert_serialization(ModelType::Seedance10Pro, "seedance_1p0_pro");
assert_serialization(ModelType::Sora2, "sora_2");
assert_serialization(ModelType::Sora2Pro, "sora_2_pro");
assert_serialization(ModelType::Veo2, "veo_2");
assert_serialization(ModelType::Veo3, "veo_3");
assert_serialization(ModelType::Veo3Fast, "veo_3_fast");
assert_serialization(ModelType::Veo3p1, "veo_3p1");
assert_serialization(ModelType::Veo3p1Fast, "veo_3p1_fast");
// 3D Object generation models
assert_serialization(ModelType::Hunyuan3d2_0, "hunyuan_3d_2p0");
assert_serialization(ModelType::Hunyuan3d2_1, "hunyuan_3d_2p1");
@@ -349,12 +379,17 @@ mod tests {
assert_eq!(ModelType::Kling16Pro.to_str(), "kling_1p6_pro");
assert_eq!(ModelType::Kling21Pro.to_str(), "kling_2p1_pro");
assert_eq!(ModelType::Kling21Master.to_str(), "kling_2p1_master");
assert_eq!(ModelType::Kling2p5TurboPro.to_str(), "kling_2p5_turbo_pro");
assert_eq!(ModelType::Kling2p6Pro.to_str(), "kling_2p6_pro");
assert_eq!(ModelType::Seedance10Lite.to_str(), "seedance_1p0_lite");
assert_eq!(ModelType::Seedance10Pro.to_str(), "seedance_1p0_pro");
assert_eq!(ModelType::Sora2.to_str(), "sora_2");
assert_eq!(ModelType::Sora2Pro.to_str(), "sora_2_pro");
assert_eq!(ModelType::Veo2.to_str(), "veo_2");
assert_eq!(ModelType::Veo3.to_str(), "veo_3");
assert_eq!(ModelType::Veo3Fast.to_str(), "veo_3_fast");
assert_eq!(ModelType::Veo3p1.to_str(), "veo_3p1");
assert_eq!(ModelType::Veo3p1Fast.to_str(), "veo_3p1_fast");
// 3D Object generation models
assert_eq!(ModelType::Hunyuan3d2_0.to_str(), "hunyuan_3d_2p0");
@@ -395,12 +430,18 @@ mod tests {
assert_eq!(ModelType::from_str("kling_1p6_pro").unwrap(), ModelType::Kling16Pro);
assert_eq!(ModelType::from_str("kling_2p1_pro").unwrap(), ModelType::Kling21Pro);
assert_eq!(ModelType::from_str("kling_2p1_master").unwrap(), ModelType::Kling21Master);
assert_eq!(ModelType::from_str("kling_2p5_turbo_pro").unwrap(), ModelType::Kling2p5TurboPro);
assert_eq!(ModelType::from_str("kling_2p6_pro").unwrap(), ModelType::Kling2p6Pro);
assert_eq!(ModelType::from_str("seedance_1p0_lite").unwrap(), ModelType::Seedance10Lite);
assert_eq!(ModelType::from_str("seedance_1p0_pro").unwrap(), ModelType::Seedance10Pro);
assert_eq!(ModelType::from_str("sora_2").unwrap(), ModelType::Sora2);
assert_eq!(ModelType::from_str("sora_2_pro").unwrap(), ModelType::Sora2Pro);
assert_eq!(ModelType::from_str("veo_2").unwrap(), ModelType::Veo2);
assert_eq!(ModelType::from_str("veo_3").unwrap(), ModelType::Veo3);
assert_eq!(ModelType::from_str("veo_3_fast").unwrap(), ModelType::Veo3Fast);
assert_eq!(ModelType::from_str("veo_3p1").unwrap(), ModelType::Veo3p1);
assert_eq!(ModelType::from_str("veo_3p1_fast").unwrap(), ModelType::Veo3p1Fast);
// 3D Object generation models
assert_eq!(ModelType::from_str("hunyuan_3d_2p0").unwrap(), ModelType::Hunyuan3d2_0);
assert_eq!(ModelType::from_str("hunyuan_3d_2p1").unwrap(), ModelType::Hunyuan3d2_1);
@@ -409,7 +450,7 @@ mod tests {
#[test]
fn all_variants() {
let mut variants = ModelType::all_variants();
assert_eq!(variants.len(), 38);
assert_eq!(variants.len(), 43);
// Image models
assert_eq!(variants.pop_first(), Some(ModelType::Flux1Dev));
assert_eq!(variants.pop_first(), Some(ModelType::Flux1Schnell));
@@ -442,12 +483,17 @@ mod tests {
assert_eq!(variants.pop_first(), Some(ModelType::Kling16Pro));
assert_eq!(variants.pop_first(), Some(ModelType::Kling21Pro));
assert_eq!(variants.pop_first(), Some(ModelType::Kling21Master));
assert_eq!(variants.pop_first(), Some(ModelType::Kling2p5TurboPro));
assert_eq!(variants.pop_first(), Some(ModelType::Kling2p6Pro));
assert_eq!(variants.pop_first(), Some(ModelType::Seedance10Lite));
assert_eq!(variants.pop_first(), Some(ModelType::Seedance10Pro));
assert_eq!(variants.pop_first(), Some(ModelType::Sora2));
assert_eq!(variants.pop_first(), Some(ModelType::Sora2Pro));
assert_eq!(variants.pop_first(), Some(ModelType::Veo2));
assert_eq!(variants.pop_first(), Some(ModelType::Veo3));
assert_eq!(variants.pop_first(), Some(ModelType::Veo3Fast));
assert_eq!(variants.pop_first(), Some(ModelType::Veo3p1));
assert_eq!(variants.pop_first(), Some(ModelType::Veo3p1Fast));
// 3D Object generation models
assert_eq!(variants.pop_first(), Some(ModelType::Hunyuan3d2_0));
assert_eq!(variants.pop_first(), Some(ModelType::Hunyuan3d2_1));
@@ -5,4 +5,5 @@ pub mod generate_seedance_1_0_lite_image_to_video_handler;
pub mod generate_seedance_1_0_pro_image_to_video_handler;
pub mod generate_veo_2_image_to_video_handler;
pub mod generate_veo_3_fast_image_to_video_handler;
pub mod generate_veo_3_image_to_video_handler;
pub mod generate_veo_3_image_to_video_handler;
pub mod multi_function;
@@ -0,0 +1,317 @@
use std::collections::{HashMap, HashSet};
use std::iter::FromIterator;
use std::sync::Arc;
use crate::http_server::common_responses::common_web_error::CommonWebError;
use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder;
use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test;
use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::state::server_state::ServerState;
use crate::util::lookup::lookup_image_urls_as_map::lookup_image_urls_as_map;
use actix_web::web::Json;
use actix_web::{web, HttpRequest};
use artcraft_api_defs::generate::video::multi_function::kling_2_5_turbo_multi_function_video_gen::{Kling2p5TurboProMultiFunctionVideoGenAspectRatio, Kling2p5TurboProMultiFunctionVideoGenDuration, Kling2p5TurboProMultiFunctionVideoGenRequest, Kling2p5TurboProMultiFunctionVideoGenResponse};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType;
use enums::by_table::prompts::prompt_type::PromptType;
use enums::common::generation_provider::GenerationProvider;
use enums::common::model_type::ModelType;
use enums::common::visibility::Visibility;
use fal_client::creds::open_ai_api_key::OpenAiApiKey;
use fal_client::requests::webhook::image::edit::enqueue_gpt_image_1p5_edit_image_webhook::{enqueue_gpt_image_1p5_image_edit_webhook, EnqueueGptImage1p5EditImageArgs, EnqueueGptImage1p5EditImageBackground, EnqueueGptImage1p5EditImageInputFidelity, EnqueueGptImage1p5EditImageNumImages, EnqueueGptImage1p5EditImageQuality, EnqueueGptImage1p5EditImageSize};
use fal_client::requests::webhook::image::text::enqueue_gpt_image_1p5_text_to_image_webhook::{enqueue_gpt_image_1p5_text_to_image_webhook, EnqueueGptImage1p5TextToImageArgs, EnqueueGptImage1p5TextToImageBackground, EnqueueGptImage1p5TextToImageNumImages, EnqueueGptImage1p5TextToImageQuality, EnqueueGptImage1p5TextToImageSize};
use fal_client::requests::webhook::video::image::enqueue_kling_v2p5_turbo_pro_image_to_video_webhook::{enqueue_kling_v2p5_turbo_pro_image_to_video_webhook, EnqueueKlingV2p5TurboProImageToVideoArgs, EnqueueKlingV2p5TurboProImageToVideoDurationSeconds};
use fal_client::requests::webhook::video::text::enqueue_kling_v2p5_turbo_pro_text_to_video_webhook::{enqueue_kling_v2p5_turbo_pro_text_to_video_webhook, EnqueueKlingV2p5TurboProTextToVideoArgs, EnqueueKlingV2p5TurboProTextToVideoAspectRatio, EnqueueKlingV2p5TurboProTextToVideoDurationSeconds};
use http_server_common::request::get_request_ip::get_request_ip;
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs;
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection};
use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem};
use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs};
use server_environment::ServerEnvironment;
use sqlx::pool::PoolConnection;
use sqlx::{Acquire, MySql};
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
/// Kling 2.5 Turbo Pro Multi-Function (text and image to video)
#[utoipa::path(
post,
tag = "Generate Video (Multi-Function)",
path = "/v1/generate/video/multi_function/kling_2p5_turbo_pro",
responses(
(status = 200, description = "Success", body = Kling2p5TurboProMultiFunctionVideoGenResponse),
),
params(
("request" = Kling2p5TurboProMultiFunctionVideoGenRequest, description = "Payload for Request"),
)
)]
pub async fn kling_2p5_turbo_pro_multi_function_video_gen_handler(
http_request: HttpRequest,
request: Json<Kling2p5TurboProMultiFunctionVideoGenRequest>,
server_state: web::Data<Arc<ServerState>>
) -> Result<Json<Kling2p5TurboProMultiFunctionVideoGenResponse>, CommonWebError> {
payments_error_test(&request.prompt.as_deref().unwrap_or(""))?;
let mut mysql_connection = server_state.mysql_pool
.acquire()
.await?;
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_from_connection(&http_request, &mut mysql_connection)
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
CommonWebError::ServerError
})?;
let maybe_avt_token = server_state
.avt_cookie_manager
.get_avt_token_from_request(&http_request);
// TODO: Limit usage for new accounts. Billing, free credits metering, etc.
//let user_session = match maybe_user_session {
// Some(session) => session,
// None => {
// warn!("not logged in");
// return Err(CommonWebError::NotAuthorized);
// }
//};
if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
return Err(CommonWebError::BadInputWithSimpleMessage(reason));
}
let mut query_media_tokens = None;
if let Some(start_frame_token) = request.start_frame_image_media_token.as_ref() {
let mut tokens = Vec::new();
tokens.push(start_frame_token.to_owned());
if let Some(end_frame_token) = request.end_frame_image_media_token.as_ref() {
// NB: Masks are only relevant when this is an image editing exercise.
tokens.push(end_frame_token.to_owned());
}
query_media_tokens = Some(tokens);
}
let image_urls_by_token = match query_media_tokens {
None => HashMap::new(),
Some(media_tokens) => {
info!("Looking up image media tokens: {:?}", media_tokens);
lookup_image_urls_as_map(
&http_request,
&mut mysql_connection,
server_state.server_environment,
&media_tokens,
).await?
}
};
let maybe_start_frame_image_url = match request.start_frame_image_media_token.as_ref() {
None => None,
Some(token) => match image_urls_by_token.get(token) {
Some(url) => Some(url.to_string()),
None => {
return Err(CommonWebError::BadInputWithSimpleMessage("Media for start frame not found.".to_string()));
}
}
};
let maybe_end_frame_image_url = match request.end_frame_image_media_token.as_ref() {
None => None,
Some(token) => match image_urls_by_token.get(token) {
Some(url) => Some(url.to_string()),
None => {
return Err(CommonWebError::BadInputWithSimpleMessage("Media for end frame not found.".to_string()));
}
}
};
insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string())
})?;
info!("Fal webhook URL: {}", server_state.fal.webhook_url);
let fal_result;
if let Some(start_frame_url) = maybe_start_frame_image_url {
info!("image-to-video case");
let duration = match request.duration {
Some(Kling2p5TurboProMultiFunctionVideoGenDuration::FiveSeconds) => EnqueueKlingV2p5TurboProImageToVideoDurationSeconds::Five,
Some(Kling2p5TurboProMultiFunctionVideoGenDuration::TenSeconds) => EnqueueKlingV2p5TurboProImageToVideoDurationSeconds::Ten,
None => EnqueueKlingV2p5TurboProImageToVideoDurationSeconds::Five,
};
let args = EnqueueKlingV2p5TurboProImageToVideoArgs {
prompt: request.prompt.as_deref().unwrap_or("").to_string(),
image_url: start_frame_url,
tail_image_url: maybe_end_frame_image_url,
negative_prompt: request.negative_prompt.clone(),
duration: Some(duration),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
fal_result = enqueue_kling_v2p5_turbo_pro_image_to_video_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_kling_v2p5_turbo_pro_image_to_video_webhook: {:?}", err);
CommonWebError::ServerError
})?;
} else {
info!("text-to-video case");
let duration = match request.duration {
Some(Kling2p5TurboProMultiFunctionVideoGenDuration::FiveSeconds) => EnqueueKlingV2p5TurboProTextToVideoDurationSeconds::Five,
Some(Kling2p5TurboProMultiFunctionVideoGenDuration::TenSeconds) => EnqueueKlingV2p5TurboProTextToVideoDurationSeconds::Ten,
None => EnqueueKlingV2p5TurboProTextToVideoDurationSeconds::Five,
};
let aspect_ratio = match request.aspect_ratio {
Some(Kling2p5TurboProMultiFunctionVideoGenAspectRatio::Square) => EnqueueKlingV2p5TurboProTextToVideoAspectRatio::Square,
Some(Kling2p5TurboProMultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueKlingV2p5TurboProTextToVideoAspectRatio::SixteenByNine,
Some(Kling2p5TurboProMultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueKlingV2p5TurboProTextToVideoAspectRatio::NineBySixteen,
None => EnqueueKlingV2p5TurboProTextToVideoAspectRatio::Square,
};
let args = EnqueueKlingV2p5TurboProTextToVideoArgs {
prompt: request.prompt.as_deref().unwrap_or("").to_string(),
negative_prompt: request.negative_prompt.clone(),
duration: Some(duration),
aspect_ratio: Some(aspect_ratio),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
fal_result = enqueue_kling_v2p5_turbo_pro_text_to_video_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_kling_v2p5_turbo_pro_text_to_video_webhook: {:?}", err);
CommonWebError::ServerError
})?;
}
let external_job_id = fal_result.request_id
.ok_or_else(|| {
warn!("Fal request_id is None");
CommonWebError::ServerError
})?;
info!("Fal request_id: {}", external_job_id);
let ip_address = get_request_ip(&http_request);
let mut transaction = mysql_connection
.begin()
.await
.map_err(|err| {
error!("Error starting MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
// NB: Don't fail the job if the query fails.
let prompt_result = insert_prompt(InsertPromptArgs {
maybe_apriori_prompt_token: None,
prompt_type: PromptType::ArtcraftApp,
maybe_creator_user_token: maybe_user_session
.as_ref()
.map(|s| &s.user_token),
maybe_model_type: Some(ModelType::Kling2p5TurboPro),
maybe_generation_provider: Some(GenerationProvider::Artcraft),
maybe_positive_prompt: request.prompt.as_deref(),
maybe_negative_prompt: None,
maybe_other_args: None,
creator_ip_address: &ip_address,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let prompt_token = match prompt_result {
Ok(token) => Some(token),
Err(err) => {
warn!("Error inserting prompt: {:?}", err);
None // Don't fail the job if the prompt insertion fails.
}
};
if let Some(token) = prompt_token.as_ref() {
let mut context_items = Vec::with_capacity(2);
if let Some(media_token) = &request.start_frame_image_media_token {
context_items.push(PromptContextItem {
media_token: media_token.clone(),
context_semantic_type: PromptContextSemanticType::VidStartFrame,
});
}
if let Some(media_token) = &request.end_frame_image_media_token {
context_items.push(PromptContextItem {
media_token: media_token.clone(),
context_semantic_type: PromptContextSemanticType::VidEndFrame,
});
}
if !context_items.is_empty() {
let result = insert_batch_prompt_context_items(InsertBatchArgs {
prompt_token: token.clone(),
items: context_items,
transaction: &mut transaction,
}).await;
if let Err(err) = result {
// NB: Fail open.
warn!("Error inserting batch prompt context items: {:?}", err);
}
}
}
let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs {
uuid_idempotency_token: &request.uuid_idempotency_token,
maybe_external_third_party_id: &external_job_id,
fal_category: FalCategory::VideoGeneration,
maybe_inference_args: None,
maybe_prompt_token: prompt_token.as_ref(),
maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: Visibility::Public,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let job_token = match db_result {
Ok(token) => token,
Err(err) => {
warn!("Error inserting generic inference job for FAL queue: {:?}", err);
return Err(CommonWebError::ServerError);
}
};
let _r = transaction
.commit()
.await
.map_err(|err| {
error!("Error committing MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
Ok(Json(Kling2p5TurboProMultiFunctionVideoGenResponse {
success: true,
inference_job_token: job_token,
}))
}
@@ -0,0 +1,316 @@
use std::collections::{HashMap, HashSet};
use std::iter::FromIterator;
use std::sync::Arc;
use crate::http_server::common_responses::common_web_error::CommonWebError;
use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder;
use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test;
use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::state::server_state::ServerState;
use crate::util::lookup::lookup_image_urls_as_map::lookup_image_urls_as_map;
use actix_web::web::Json;
use actix_web::{web, HttpRequest};
use artcraft_api_defs::generate::video::multi_function::kling_2_6_multi_function_video_gen::{Kling2p6ProMultiFunctionVideoGenAspectRatio, Kling2p6ProMultiFunctionVideoGenDuration, Kling2p6ProMultiFunctionVideoGenRequest, Kling2p6ProMultiFunctionVideoGenResponse};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType;
use enums::by_table::prompts::prompt_type::PromptType;
use enums::common::generation_provider::GenerationProvider;
use enums::common::model_type::ModelType;
use enums::common::visibility::Visibility;
use fal_client::creds::open_ai_api_key::OpenAiApiKey;
use fal_client::requests::webhook::video::image::enqueue_kling_v2p6_pro_image_to_video_webhook::{enqueue_kling_v2p6_pro_image_to_video_webhook, EnqueueKlingV2p6ProImageToVideoArgs, EnqueueKlingV2p6ProImageToVideoDurationSeconds};
use fal_client::requests::webhook::video::text::enqueue_kling_v2p6_pro_text_to_video_webhook::{enqueue_kling_v2p6_pro_text_to_video_webhook, EnqueueKlingV2p6ProTextToVideoArgs, EnqueueKlingV2p6ProTextToVideoAspectRatio, EnqueueKlingV2p6ProTextToVideoDurationSeconds};
use http_server_common::request::get_request_ip::get_request_ip;
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs;
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection};
use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem};
use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs};
use server_environment::ServerEnvironment;
use sqlx::pool::PoolConnection;
use sqlx::{Acquire, MySql};
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
/// Kling 2.6 Pro Multi-Function (text and image to video)
#[utoipa::path(
post,
tag = "Generate Video (Multi-Function)",
path = "/v1/generate/video/multi_function/kling_2p6_pro",
responses(
(status = 200, description = "Success", body = Kling2p6ProMultiFunctionVideoGenResponse),
),
params(
("request" = Kling2p6ProMultiFunctionVideoGenRequest, description = "Payload for Request"),
)
)]
pub async fn kling_2p6_pro_multi_function_video_gen_handler(
http_request: HttpRequest,
request: Json<Kling2p6ProMultiFunctionVideoGenRequest>,
server_state: web::Data<Arc<ServerState>>
) -> Result<Json<Kling2p6ProMultiFunctionVideoGenResponse>, CommonWebError> {
payments_error_test(&request.prompt.as_deref().unwrap_or(""))?;
let mut mysql_connection = server_state.mysql_pool
.acquire()
.await?;
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_from_connection(&http_request, &mut mysql_connection)
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
CommonWebError::ServerError
})?;
let maybe_avt_token = server_state
.avt_cookie_manager
.get_avt_token_from_request(&http_request);
// TODO: Limit usage for new accounts. Billing, free credits metering, etc.
//let user_session = match maybe_user_session {
// Some(session) => session,
// None => {
// warn!("not logged in");
// return Err(CommonWebError::NotAuthorized);
// }
//};
if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
return Err(CommonWebError::BadInputWithSimpleMessage(reason));
}
let mut query_media_tokens = None;
if let Some(start_frame_token) = request.start_frame_image_media_token.as_ref() {
let mut tokens = Vec::new();
tokens.push(start_frame_token.to_owned());
// if let Some(end_frame_token) = request.end_frame_image_media_token.as_ref() {
// // NB: Masks are only relevant when this is an image editing exercise.
// tokens.push(end_frame_token.to_owned());
// }
query_media_tokens = Some(tokens);
}
let image_urls_by_token = match query_media_tokens {
None => HashMap::new(),
Some(media_tokens) => {
info!("Looking up image media tokens: {:?}", media_tokens);
lookup_image_urls_as_map(
&http_request,
&mut mysql_connection,
server_state.server_environment,
&media_tokens,
).await?
}
};
let maybe_start_frame_image_url = match request.start_frame_image_media_token.as_ref() {
None => None,
Some(token) => match image_urls_by_token.get(token) {
Some(url) => Some(url.to_string()),
None => {
return Err(CommonWebError::BadInputWithSimpleMessage("Media for start frame not found.".to_string()));
}
}
};
// let maybe_end_frame_image_url = match request.end_frame_image_media_token.as_ref() {
// None => None,
// Some(token) => match image_urls_by_token.get(token) {
// Some(url) => Some(url.to_string()),
// None => {
// return Err(CommonWebError::BadInputWithSimpleMessage("Media for end frame not found.".to_string()));
// }
// }
// };
insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string())
})?;
info!("Fal webhook URL: {}", server_state.fal.webhook_url);
let fal_result;
if let Some(start_frame_url) = maybe_start_frame_image_url {
info!("image-to-video case");
let duration = match request.duration {
Some(Kling2p6ProMultiFunctionVideoGenDuration::FiveSeconds) => EnqueueKlingV2p6ProImageToVideoDurationSeconds::Five,
Some(Kling2p6ProMultiFunctionVideoGenDuration::TenSeconds) => EnqueueKlingV2p6ProImageToVideoDurationSeconds::Ten,
None => EnqueueKlingV2p6ProImageToVideoDurationSeconds::Five,
};
let args = EnqueueKlingV2p6ProImageToVideoArgs {
prompt: request.prompt.as_deref().unwrap_or("").to_string(),
image_url: start_frame_url,
generate_audio: request.generate_audio,
negative_prompt: request.negative_prompt.clone(),
duration: Some(duration),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
fal_result = enqueue_kling_v2p6_pro_image_to_video_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_kling_v2p6_pro_image_to_video_webhook: {:?}", err);
CommonWebError::ServerError
})?;
} else {
info!("text-to-video case");
let duration = match request.duration {
Some(Kling2p6ProMultiFunctionVideoGenDuration::FiveSeconds) => EnqueueKlingV2p6ProTextToVideoDurationSeconds::Five,
Some(Kling2p6ProMultiFunctionVideoGenDuration::TenSeconds) => EnqueueKlingV2p6ProTextToVideoDurationSeconds::Ten,
None => EnqueueKlingV2p6ProTextToVideoDurationSeconds::Five,
};
let aspect_ratio = match request.aspect_ratio {
Some(Kling2p6ProMultiFunctionVideoGenAspectRatio::Square) => EnqueueKlingV2p6ProTextToVideoAspectRatio::Square,
Some(Kling2p6ProMultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueKlingV2p6ProTextToVideoAspectRatio::SixteenByNine,
Some(Kling2p6ProMultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueKlingV2p6ProTextToVideoAspectRatio::NineBySixteen,
None => EnqueueKlingV2p6ProTextToVideoAspectRatio::Square,
};
let args = EnqueueKlingV2p6ProTextToVideoArgs {
prompt: request.prompt.as_deref().unwrap_or("").to_string(),
negative_prompt: request.negative_prompt.clone(),
generate_audio: request.generate_audio,
duration: Some(duration),
aspect_ratio: Some(aspect_ratio),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
fal_result = enqueue_kling_v2p6_pro_text_to_video_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_kling_v2p6_pro_text_to_video_webhook: {:?}", err);
CommonWebError::ServerError
})?;
}
let external_job_id = fal_result.request_id
.ok_or_else(|| {
warn!("Fal request_id is None");
CommonWebError::ServerError
})?;
info!("Fal request_id: {}", external_job_id);
let ip_address = get_request_ip(&http_request);
let mut transaction = mysql_connection
.begin()
.await
.map_err(|err| {
error!("Error starting MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
// NB: Don't fail the job if the query fails.
let prompt_result = insert_prompt(InsertPromptArgs {
maybe_apriori_prompt_token: None,
prompt_type: PromptType::ArtcraftApp,
maybe_creator_user_token: maybe_user_session
.as_ref()
.map(|s| &s.user_token),
maybe_model_type: Some(ModelType::Kling2p6Pro),
maybe_generation_provider: Some(GenerationProvider::Artcraft),
maybe_positive_prompt: request.prompt.as_deref(),
maybe_negative_prompt: None,
maybe_other_args: None,
creator_ip_address: &ip_address,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let prompt_token = match prompt_result {
Ok(token) => Some(token),
Err(err) => {
warn!("Error inserting prompt: {:?}", err);
None // Don't fail the job if the prompt insertion fails.
}
};
if let Some(token) = prompt_token.as_ref() {
let mut context_items = Vec::with_capacity(2);
if let Some(media_token) = &request.start_frame_image_media_token {
context_items.push(PromptContextItem {
media_token: media_token.clone(),
context_semantic_type: PromptContextSemanticType::VidStartFrame,
});
}
// if let Some(media_token) = &request.end_frame_image_media_token {
// context_items.push(PromptContextItem {
// media_token: media_token.clone(),
// context_semantic_type: PromptContextSemanticType::VidEndFrame,
// });
// }
if !context_items.is_empty() {
let result = insert_batch_prompt_context_items(InsertBatchArgs {
prompt_token: token.clone(),
items: context_items,
transaction: &mut transaction,
}).await;
if let Err(err) = result {
// NB: Fail open.
warn!("Error inserting batch prompt context items: {:?}", err);
}
}
}
let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs {
uuid_idempotency_token: &request.uuid_idempotency_token,
maybe_external_third_party_id: &external_job_id,
fal_category: FalCategory::VideoGeneration,
maybe_inference_args: None,
maybe_prompt_token: prompt_token.as_ref(),
maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: Visibility::Public,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let job_token = match db_result {
Ok(token) => token,
Err(err) => {
warn!("Error inserting generic inference job for FAL queue: {:?}", err);
return Err(CommonWebError::ServerError);
}
};
let _r = transaction
.commit()
.await
.map_err(|err| {
error!("Error committing MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
Ok(Json(Kling2p6ProMultiFunctionVideoGenResponse {
success: true,
inference_job_token: job_token,
}))
}
@@ -0,0 +1,6 @@
pub mod kling_2p5_turbo_pro_multi_function_video_gen_handler;
pub mod kling_2p6_pro_multi_function_video_gen_handler;
pub mod sora_2_multi_function_video_gen_handler;
pub mod sora_2_pro_multi_function_video_gen_handler;
pub mod veo_3p1_fast_multi_function_video_gen_handler;
pub mod veo_3p1_multi_function_video_gen_handler;
@@ -0,0 +1,336 @@
use std::collections::{HashMap, HashSet};
use std::iter::FromIterator;
use std::sync::Arc;
use crate::http_server::common_responses::common_web_error::CommonWebError;
use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder;
use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test;
use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::state::server_state::ServerState;
use crate::util::lookup::lookup_image_urls_as_map::lookup_image_urls_as_map;
use actix_web::web::Json;
use actix_web::{web, HttpRequest};
use artcraft_api_defs::generate::video::multi_function::sora_2_multi_function_video_gen::{Sora2MultiFunctionVideoGenAspectRatio, Sora2MultiFunctionVideoGenDuration, Sora2MultiFunctionVideoGenRequest, Sora2MultiFunctionVideoGenResolution, Sora2MultiFunctionVideoGenResponse};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType;
use enums::by_table::prompts::prompt_type::PromptType;
use enums::common::generation_provider::GenerationProvider;
use enums::common::model_type::ModelType;
use enums::common::visibility::Visibility;
use fal_client::creds::open_ai_api_key::OpenAiApiKey;
use fal_client::requests::webhook::video::image::enqueue_sora_2_image_to_video_webhook::{enqueue_sora_2_image_to_video_webhook, EnqueueSora2ImageToVideoArgs, EnqueueSora2ImageToVideoAspectRatio, EnqueueSora2ImageToVideoDurationSeconds, EnqueueSora2ImageToVideoResolution};
use fal_client::requests::webhook::video::text::enqueue_sora_2_text_to_video_webhook::{enqueue_sora_2_text_to_video_webhook, EnqueueSora2TextToVideoArgs, EnqueueSora2TextToVideoAspectRatio, EnqueueSora2TextToVideoDurationSeconds, EnqueueSora2TextToVideoResolution};
use http_server_common::request::get_request_ip::get_request_ip;
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs;
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection};
use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem};
use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs};
use server_environment::ServerEnvironment;
use sqlx::pool::PoolConnection;
use sqlx::{Acquire, MySql};
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
/// Sora 2 Multi-Function (text and image to video)
#[utoipa::path(
post,
tag = "Generate Video (Multi-Function)",
path = "/v1/generate/video/multi_function/sora_2",
responses(
(status = 200, description = "Success", body = Sora2MultiFunctionVideoGenResponse),
),
params(
("request" = Sora2MultiFunctionVideoGenRequest, description = "Payload for Request"),
)
)]
pub async fn sora_2_multi_function_video_gen_handler(
http_request: HttpRequest,
request: Json<Sora2MultiFunctionVideoGenRequest>,
server_state: web::Data<Arc<ServerState>>
) -> Result<Json<Sora2MultiFunctionVideoGenResponse>, CommonWebError> {
payments_error_test(&request.prompt.as_deref().unwrap_or(""))?;
let mut mysql_connection = server_state.mysql_pool
.acquire()
.await?;
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_from_connection(&http_request, &mut mysql_connection)
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
CommonWebError::ServerError
})?;
let maybe_avt_token = server_state
.avt_cookie_manager
.get_avt_token_from_request(&http_request);
// TODO: Limit usage for new accounts. Billing, free credits metering, etc.
//let user_session = match maybe_user_session {
// Some(session) => session,
// None => {
// warn!("not logged in");
// return Err(CommonWebError::NotAuthorized);
// }
//};
if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
return Err(CommonWebError::BadInputWithSimpleMessage(reason));
}
let mut query_media_tokens = None;
if let Some(start_frame_token) = request.image_media_token.as_ref() {
let mut tokens = Vec::new();
tokens.push(start_frame_token.to_owned());
// if let Some(end_frame_token) = request.end_frame_image_media_token.as_ref() {
// // NB: Masks are only relevant when this is an image editing exercise.
// tokens.push(end_frame_token.to_owned());
// }
query_media_tokens = Some(tokens);
}
let image_urls_by_token = match query_media_tokens {
None => HashMap::new(),
Some(media_tokens) => {
info!("Looking up image media tokens: {:?}", media_tokens);
lookup_image_urls_as_map(
&http_request,
&mut mysql_connection,
server_state.server_environment,
&media_tokens,
).await?
}
};
let maybe_image_url = match request.image_media_token.as_ref() {
None => None,
Some(token) => match image_urls_by_token.get(token) {
Some(url) => Some(url.to_string()),
None => {
return Err(CommonWebError::BadInputWithSimpleMessage("Media for start frame not found.".to_string()));
}
}
};
// let maybe_end_frame_image_url = match request.end_frame_image_media_token.as_ref() {
// None => None,
// Some(token) => match image_urls_by_token.get(token) {
// Some(url) => Some(url.to_string()),
// None => {
// return Err(CommonWebError::BadInputWithSimpleMessage("Media for end frame not found.".to_string()));
// }
// }
// };
insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string())
})?;
info!("Fal webhook URL: {}", server_state.fal.webhook_url);
let fal_result;
if let Some(image_url) = maybe_image_url {
info!("image-to-video case");
let duration = match request.duration {
Some(Sora2MultiFunctionVideoGenDuration::FourSeconds) => EnqueueSora2ImageToVideoDurationSeconds::Four,
Some(Sora2MultiFunctionVideoGenDuration::EightSeconds) => EnqueueSora2ImageToVideoDurationSeconds::Eight,
Some(Sora2MultiFunctionVideoGenDuration::TwelveSeconds) => EnqueueSora2ImageToVideoDurationSeconds::Twelve,
None => EnqueueSora2ImageToVideoDurationSeconds::Eight,
};
let aspect_ratio = match request.aspect_ratio {
Some(Sora2MultiFunctionVideoGenAspectRatio::Auto) => EnqueueSora2ImageToVideoAspectRatio::Auto,
Some(Sora2MultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueSora2ImageToVideoAspectRatio::SixteenByNine,
Some(Sora2MultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueSora2ImageToVideoAspectRatio::NineBySixteen,
None => EnqueueSora2ImageToVideoAspectRatio::SixteenByNine,
};
let resolution = match request.resolution {
Some(Sora2MultiFunctionVideoGenResolution::Auto) => EnqueueSora2ImageToVideoResolution::Auto,
Some(Sora2MultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueSora2ImageToVideoResolution::SevenTwentyP,
None => EnqueueSora2ImageToVideoResolution::SevenTwentyP,
};
let args = EnqueueSora2ImageToVideoArgs {
image_url,
prompt: request.prompt.as_deref().unwrap_or("").to_string(),
duration: Some(duration),
aspect_ratio: Some(aspect_ratio),
resolution: Some(resolution),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
fal_result = enqueue_sora_2_image_to_video_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_sora_2_image_to_video_webhook: {:?}", err);
CommonWebError::ServerError
})?;
} else {
info!("text-to-video case");
let duration = match request.duration {
Some(Sora2MultiFunctionVideoGenDuration::FourSeconds) => EnqueueSora2TextToVideoDurationSeconds::Four,
Some(Sora2MultiFunctionVideoGenDuration::EightSeconds) => EnqueueSora2TextToVideoDurationSeconds::Eight,
Some(Sora2MultiFunctionVideoGenDuration::TwelveSeconds) => EnqueueSora2TextToVideoDurationSeconds::Twelve,
None => EnqueueSora2TextToVideoDurationSeconds::Eight,
};
let aspect_ratio = match request.aspect_ratio {
Some(Sora2MultiFunctionVideoGenAspectRatio::Auto) => EnqueueSora2TextToVideoAspectRatio::Auto,
Some(Sora2MultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueSora2TextToVideoAspectRatio::SixteenByNine,
Some(Sora2MultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueSora2TextToVideoAspectRatio::NineBySixteen,
None => EnqueueSora2TextToVideoAspectRatio::SixteenByNine,
};
let resolution = match request.resolution {
Some(Sora2MultiFunctionVideoGenResolution::Auto) => EnqueueSora2TextToVideoResolution::Auto,
Some(Sora2MultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueSora2TextToVideoResolution::SevenTwentyP,
None => EnqueueSora2TextToVideoResolution::SevenTwentyP,
};
let args = EnqueueSora2TextToVideoArgs {
prompt: request.prompt.as_deref().unwrap_or("").to_string(),
duration: Some(duration),
aspect_ratio: Some(aspect_ratio),
resolution: Some(resolution),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
fal_result = enqueue_sora_2_text_to_video_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_sora_2_text_to_video_webhook: {:?}", err);
CommonWebError::ServerError
})?;
}
let external_job_id = fal_result.request_id
.ok_or_else(|| {
warn!("Fal request_id is None");
CommonWebError::ServerError
})?;
info!("Fal request_id: {}", external_job_id);
let ip_address = get_request_ip(&http_request);
let mut transaction = mysql_connection
.begin()
.await
.map_err(|err| {
error!("Error starting MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
// NB: Don't fail the job if the query fails.
let prompt_result = insert_prompt(InsertPromptArgs {
maybe_apriori_prompt_token: None,
prompt_type: PromptType::ArtcraftApp,
maybe_creator_user_token: maybe_user_session
.as_ref()
.map(|s| &s.user_token),
maybe_model_type: Some(ModelType::Sora2),
maybe_generation_provider: Some(GenerationProvider::Artcraft),
maybe_positive_prompt: request.prompt.as_deref(),
maybe_negative_prompt: None,
maybe_other_args: None,
creator_ip_address: &ip_address,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let prompt_token = match prompt_result {
Ok(token) => Some(token),
Err(err) => {
warn!("Error inserting prompt: {:?}", err);
None // Don't fail the job if the prompt insertion fails.
}
};
if let Some(token) = prompt_token.as_ref() {
let mut context_items = Vec::with_capacity(2);
if let Some(media_token) = &request.image_media_token {
context_items.push(PromptContextItem {
media_token: media_token.clone(),
context_semantic_type: PromptContextSemanticType::VidStartFrame,
});
}
// if let Some(media_token) = &request.end_frame_image_media_token {
// context_items.push(PromptContextItem {
// media_token: media_token.clone(),
// context_semantic_type: PromptContextSemanticType::VidEndFrame,
// });
// }
if !context_items.is_empty() {
let result = insert_batch_prompt_context_items(InsertBatchArgs {
prompt_token: token.clone(),
items: context_items,
transaction: &mut transaction,
}).await;
if let Err(err) = result {
// NB: Fail open.
warn!("Error inserting batch prompt context items: {:?}", err);
}
}
}
let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs {
uuid_idempotency_token: &request.uuid_idempotency_token,
maybe_external_third_party_id: &external_job_id,
fal_category: FalCategory::VideoGeneration,
maybe_inference_args: None,
maybe_prompt_token: prompt_token.as_ref(),
maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: Visibility::Public,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let job_token = match db_result {
Ok(token) => token,
Err(err) => {
warn!("Error inserting generic inference job for FAL queue: {:?}", err);
return Err(CommonWebError::ServerError);
}
};
let _r = transaction
.commit()
.await
.map_err(|err| {
error!("Error committing MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
Ok(Json(Sora2MultiFunctionVideoGenResponse {
success: true,
inference_job_token: job_token,
}))
}
@@ -0,0 +1,338 @@
use std::collections::{HashMap, HashSet};
use std::iter::FromIterator;
use std::sync::Arc;
use crate::http_server::common_responses::common_web_error::CommonWebError;
use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder;
use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test;
use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::state::server_state::ServerState;
use crate::util::lookup::lookup_image_urls_as_map::lookup_image_urls_as_map;
use actix_web::web::Json;
use actix_web::{web, HttpRequest};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType;
use enums::by_table::prompts::prompt_type::PromptType;
use enums::common::generation_provider::GenerationProvider;
use enums::common::model_type::ModelType;
use enums::common::visibility::Visibility;
use fal_client::creds::open_ai_api_key::OpenAiApiKey;
use http_server_common::request::get_request_ip::get_request_ip;
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs;
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection};
use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem};
use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs};
use server_environment::ServerEnvironment;
use sqlx::pool::PoolConnection;
use sqlx::{Acquire, MySql};
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
use artcraft_api_defs::generate::video::multi_function::sora_2_pro_multi_function_video_gen::{Sora2ProMultiFunctionVideoGenAspectRatio, Sora2ProMultiFunctionVideoGenDuration, Sora2ProMultiFunctionVideoGenRequest, Sora2ProMultiFunctionVideoGenResolution, Sora2ProMultiFunctionVideoGenResponse};
use fal_client::requests::webhook::video::image::enqueue_sora_2_pro_image_to_video_webhook::{enqueue_sora_2_pro_image_to_video_webhook, EnqueueSora2ProImageToVideoArgs, EnqueueSora2ProImageToVideoAspectRatio, EnqueueSora2ProImageToVideoDurationSeconds, EnqueueSora2ProImageToVideoResolution};
use fal_client::requests::webhook::video::text::enqueue_sora_2_pro_text_to_video_webhook::{enqueue_sora_2_pro_text_to_video_webhook, EnqueueSora2ProTextToVideoArgs, EnqueueSora2ProTextToVideoAspectRatio, EnqueueSora2ProTextToVideoDurationSeconds, EnqueueSora2ProTextToVideoResolution};
/// Sora 2 Pro Multi-Function (text and image to video)
#[utoipa::path(
post,
tag = "Generate Video (Multi-Function)",
path = "/v1/generate/video/multi_function/sora_2_pro",
responses(
(status = 200, description = "Success", body = Sora2ProMultiFunctionVideoGenResponse),
),
params(
("request" = Sora2ProMultiFunctionVideoGenRequest, description = "Payload for Request"),
)
)]
pub async fn sora_2_pro_multi_function_video_gen_handler(
http_request: HttpRequest,
request: Json<Sora2ProMultiFunctionVideoGenRequest>,
server_state: web::Data<Arc<ServerState>>
) -> Result<Json<Sora2ProMultiFunctionVideoGenResponse>, CommonWebError> {
payments_error_test(&request.prompt.as_deref().unwrap_or(""))?;
let mut mysql_connection = server_state.mysql_pool
.acquire()
.await?;
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_from_connection(&http_request, &mut mysql_connection)
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
CommonWebError::ServerError
})?;
let maybe_avt_token = server_state
.avt_cookie_manager
.get_avt_token_from_request(&http_request);
// TODO: Limit usage for new accounts. Billing, free credits metering, etc.
//let user_session = match maybe_user_session {
// Some(session) => session,
// None => {
// warn!("not logged in");
// return Err(CommonWebError::NotAuthorized);
// }
//};
if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
return Err(CommonWebError::BadInputWithSimpleMessage(reason));
}
let mut query_media_tokens = None;
if let Some(start_frame_token) = request.image_media_token.as_ref() {
let mut tokens = Vec::new();
tokens.push(start_frame_token.to_owned());
// if let Some(end_frame_token) = request.end_frame_image_media_token.as_ref() {
// // NB: Masks are only relevant when this is an image editing exercise.
// tokens.push(end_frame_token.to_owned());
// }
query_media_tokens = Some(tokens);
}
let image_urls_by_token = match query_media_tokens {
None => HashMap::new(),
Some(media_tokens) => {
info!("Looking up image media tokens: {:?}", media_tokens);
lookup_image_urls_as_map(
&http_request,
&mut mysql_connection,
server_state.server_environment,
&media_tokens,
).await?
}
};
let maybe_image_url = match request.image_media_token.as_ref() {
None => None,
Some(token) => match image_urls_by_token.get(token) {
Some(url) => Some(url.to_string()),
None => {
return Err(CommonWebError::BadInputWithSimpleMessage("Media for start frame not found.".to_string()));
}
}
};
// let maybe_end_frame_image_url = match request.end_frame_image_media_token.as_ref() {
// None => None,
// Some(token) => match image_urls_by_token.get(token) {
// Some(url) => Some(url.to_string()),
// None => {
// return Err(CommonWebError::BadInputWithSimpleMessage("Media for end frame not found.".to_string()));
// }
// }
// };
insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string())
})?;
info!("Fal webhook URL: {}", server_state.fal.webhook_url);
let fal_result;
if let Some(image_url) = maybe_image_url {
info!("image-to-video case");
let duration = match request.duration {
Some(Sora2ProMultiFunctionVideoGenDuration::FourSeconds) => EnqueueSora2ProImageToVideoDurationSeconds::Four,
Some(Sora2ProMultiFunctionVideoGenDuration::EightSeconds) => EnqueueSora2ProImageToVideoDurationSeconds::Eight,
Some(Sora2ProMultiFunctionVideoGenDuration::TwelveSeconds) => EnqueueSora2ProImageToVideoDurationSeconds::Twelve,
None => EnqueueSora2ProImageToVideoDurationSeconds::Eight,
};
let aspect_ratio = match request.aspect_ratio {
Some(Sora2ProMultiFunctionVideoGenAspectRatio::Auto) => EnqueueSora2ProImageToVideoAspectRatio::Auto,
Some(Sora2ProMultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueSora2ProImageToVideoAspectRatio::SixteenByNine,
Some(Sora2ProMultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueSora2ProImageToVideoAspectRatio::NineBySixteen,
None => EnqueueSora2ProImageToVideoAspectRatio::SixteenByNine,
};
let resolution = match request.resolution {
Some(Sora2ProMultiFunctionVideoGenResolution::Auto) => EnqueueSora2ProImageToVideoResolution::Auto,
Some(Sora2ProMultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueSora2ProImageToVideoResolution::SevenTwentyP,
Some(Sora2ProMultiFunctionVideoGenResolution::TenEightyP) => EnqueueSora2ProImageToVideoResolution::TenEightyP,
None => EnqueueSora2ProImageToVideoResolution::TenEightyP,
};
let args = EnqueueSora2ProImageToVideoArgs {
image_url,
prompt: request.prompt.as_deref().unwrap_or("").to_string(),
duration: Some(duration),
aspect_ratio: Some(aspect_ratio),
resolution: Some(resolution),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
fal_result = enqueue_sora_2_pro_image_to_video_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_sora_2_pro_image_to_video_webhook: {:?}", err);
CommonWebError::ServerError
})?;
} else {
info!("text-to-video case");
let duration = match request.duration {
Some(Sora2ProMultiFunctionVideoGenDuration::FourSeconds) => EnqueueSora2ProTextToVideoDurationSeconds::Four,
Some(Sora2ProMultiFunctionVideoGenDuration::EightSeconds) => EnqueueSora2ProTextToVideoDurationSeconds::Eight,
Some(Sora2ProMultiFunctionVideoGenDuration::TwelveSeconds) => EnqueueSora2ProTextToVideoDurationSeconds::Twelve,
None => EnqueueSora2ProTextToVideoDurationSeconds::Eight,
};
let aspect_ratio = match request.aspect_ratio {
Some(Sora2ProMultiFunctionVideoGenAspectRatio::Auto) => EnqueueSora2ProTextToVideoAspectRatio::Auto,
Some(Sora2ProMultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueSora2ProTextToVideoAspectRatio::SixteenByNine,
Some(Sora2ProMultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueSora2ProTextToVideoAspectRatio::NineBySixteen,
None => EnqueueSora2ProTextToVideoAspectRatio::SixteenByNine,
};
let resolution = match request.resolution {
Some(Sora2ProMultiFunctionVideoGenResolution::Auto) => EnqueueSora2ProTextToVideoResolution::Auto,
Some(Sora2ProMultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueSora2ProTextToVideoResolution::SevenTwentyP,
Some(Sora2ProMultiFunctionVideoGenResolution::TenEightyP) => EnqueueSora2ProTextToVideoResolution::TenEightyP,
None => EnqueueSora2ProTextToVideoResolution::TenEightyP,
};
let args = EnqueueSora2ProTextToVideoArgs {
prompt: request.prompt.as_deref().unwrap_or("").to_string(),
duration: Some(duration),
aspect_ratio: Some(aspect_ratio),
resolution: Some(resolution),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
fal_result = enqueue_sora_2_pro_text_to_video_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_sora_2_pro_text_to_video_webhook: {:?}", err);
CommonWebError::ServerError
})?;
}
let external_job_id = fal_result.request_id
.ok_or_else(|| {
warn!("Fal request_id is None");
CommonWebError::ServerError
})?;
info!("Fal request_id: {}", external_job_id);
let ip_address = get_request_ip(&http_request);
let mut transaction = mysql_connection
.begin()
.await
.map_err(|err| {
error!("Error starting MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
// NB: Don't fail the job if the query fails.
let prompt_result = insert_prompt(InsertPromptArgs {
maybe_apriori_prompt_token: None,
prompt_type: PromptType::ArtcraftApp,
maybe_creator_user_token: maybe_user_session
.as_ref()
.map(|s| &s.user_token),
maybe_model_type: Some(ModelType::Sora2Pro),
maybe_generation_provider: Some(GenerationProvider::Artcraft),
maybe_positive_prompt: request.prompt.as_deref(),
maybe_negative_prompt: None,
maybe_other_args: None,
creator_ip_address: &ip_address,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let prompt_token = match prompt_result {
Ok(token) => Some(token),
Err(err) => {
warn!("Error inserting prompt: {:?}", err);
None // Don't fail the job if the prompt insertion fails.
}
};
if let Some(token) = prompt_token.as_ref() {
let mut context_items = Vec::with_capacity(2);
if let Some(media_token) = &request.image_media_token {
context_items.push(PromptContextItem {
media_token: media_token.clone(),
context_semantic_type: PromptContextSemanticType::VidStartFrame,
});
}
// if let Some(media_token) = &request.end_frame_image_media_token {
// context_items.push(PromptContextItem {
// media_token: media_token.clone(),
// context_semantic_type: PromptContextSemanticType::VidEndFrame,
// });
// }
if !context_items.is_empty() {
let result = insert_batch_prompt_context_items(InsertBatchArgs {
prompt_token: token.clone(),
items: context_items,
transaction: &mut transaction,
}).await;
if let Err(err) = result {
// NB: Fail open.
warn!("Error inserting batch prompt context items: {:?}", err);
}
}
}
let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs {
uuid_idempotency_token: &request.uuid_idempotency_token,
maybe_external_third_party_id: &external_job_id,
fal_category: FalCategory::VideoGeneration,
maybe_inference_args: None,
maybe_prompt_token: prompt_token.as_ref(),
maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: Visibility::Public,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let job_token = match db_result {
Ok(token) => token,
Err(err) => {
warn!("Error inserting generic inference job for FAL queue: {:?}", err);
return Err(CommonWebError::ServerError);
}
};
let _r = transaction
.commit()
.await
.map_err(|err| {
error!("Error committing MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
Ok(Json(Sora2ProMultiFunctionVideoGenResponse {
success: true,
inference_job_token: job_token,
}))
}
@@ -0,0 +1,390 @@
use std::collections::{HashMap, HashSet};
use std::iter::FromIterator;
use std::sync::Arc;
use crate::http_server::common_responses::common_web_error::CommonWebError;
use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder;
use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test;
use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::state::server_state::ServerState;
use crate::util::lookup::lookup_image_urls_as_map::lookup_image_urls_as_map;
use actix_web::web::Json;
use actix_web::{web, HttpRequest};
use artcraft_api_defs::generate::video::multi_function::veo_3p1_fast_multi_function_video_gen::{Veo3p1FastMultiFunctionVideoGenAspectRatio, Veo3p1FastMultiFunctionVideoGenDuration, Veo3p1FastMultiFunctionVideoGenRequest, Veo3p1FastMultiFunctionVideoGenResolution, Veo3p1FastMultiFunctionVideoGenResponse};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType;
use enums::by_table::prompts::prompt_type::PromptType;
use enums::common::generation_provider::GenerationProvider;
use enums::common::model_type::ModelType;
use enums::common::visibility::Visibility;
use fal_client::creds::open_ai_api_key::OpenAiApiKey;
use fal_client::requests::webhook::video::image::enqueue_veo_3p1_fast_first_last_frame_image_to_video_webhook::{enqueue_veo_3p1_fast_first_last_frame_image_to_video_webhook, EnqueueVeo3p1FastFirstLastFrameImageToVideoArgs, EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio, EnqueueVeo3p1FastFirstLastFrameImageToVideoDurationSeconds, EnqueueVeo3p1FastFirstLastFrameImageToVideoResolution};
use fal_client::requests::webhook::video::image::enqueue_veo_3p1_fast_image_to_video_webhook::{enqueue_veo_3p1_fast_image_to_video_webhook, EnqueueVeo3p1FastImageToVideoArgs, EnqueueVeo3p1FastImageToVideoAspectRatio, EnqueueVeo3p1FastImageToVideoDurationSeconds, EnqueueVeo3p1FastImageToVideoResolution};
use fal_client::requests::webhook::video::text::enqueue_veo_3p1_fast_text_to_video_webhook::{enqueue_veo_3p1_fast_text_to_video_webhook, EnqueueVeo3p1FastTextToVideoArgs, EnqueueVeo3p1FastTextToVideoAspectRatio, EnqueueVeo3p1FastTextToVideoDurationSeconds, EnqueueVeo3p1FastTextToVideoResolution};
use http_server_common::request::get_request_ip::get_request_ip;
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs;
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection};
use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem};
use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs};
use server_environment::ServerEnvironment;
use sqlx::pool::PoolConnection;
use sqlx::{Acquire, MySql};
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
/// Veo 3.1 Fast Multi-Function (text and image to video)
#[utoipa::path(
post,
tag = "Generate Video (Multi-Function)",
path = "/v1/generate/video/multi_function/veo_3p1_fast",
responses(
(status = 200, description = "Success", body = Veo3p1FastMultiFunctionVideoGenResponse),
),
params(
("request" = Veo3p1FastMultiFunctionVideoGenRequest, description = "Payload for Request"),
)
)]
pub async fn veo_3p1_fast_multi_function_video_gen_handler(
http_request: HttpRequest,
request: Json<Veo3p1FastMultiFunctionVideoGenRequest>,
server_state: web::Data<Arc<ServerState>>
) -> Result<Json<Veo3p1FastMultiFunctionVideoGenResponse>, CommonWebError> {
payments_error_test(&request.prompt.as_deref().unwrap_or(""))?;
let mut mysql_connection = server_state.mysql_pool
.acquire()
.await?;
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_from_connection(&http_request, &mut mysql_connection)
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
CommonWebError::ServerError
})?;
let maybe_avt_token = server_state
.avt_cookie_manager
.get_avt_token_from_request(&http_request);
// TODO: Limit usage for new accounts. Billing, free credits metering, etc.
//let user_session = match maybe_user_session {
// Some(session) => session,
// None => {
// warn!("not logged in");
// return Err(CommonWebError::NotAuthorized);
// }
//};
if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
return Err(CommonWebError::BadInputWithSimpleMessage(reason));
}
let mut query_media_tokens = None;
if let Some(start_frame_token) = request.start_frame_image_media_token.as_ref() {
let mut tokens = Vec::new();
tokens.push(start_frame_token.to_owned());
if let Some(end_frame_token) = request.end_frame_image_media_token.as_ref() {
// NB: Masks are only relevant when this is an image editing exercise.
tokens.push(end_frame_token.to_owned());
}
query_media_tokens = Some(tokens);
}
let image_urls_by_token = match query_media_tokens {
None => HashMap::new(),
Some(media_tokens) => {
info!("Looking up image media tokens: {:?}", media_tokens);
lookup_image_urls_as_map(
&http_request,
&mut mysql_connection,
server_state.server_environment,
&media_tokens,
).await?
}
};
let maybe_start_frame_image_url = match request.start_frame_image_media_token.as_ref() {
None => None,
Some(token) => match image_urls_by_token.get(token) {
Some(url) => Some(url.to_string()),
None => {
return Err(CommonWebError::BadInputWithSimpleMessage("Media for start frame not found.".to_string()));
}
}
};
let maybe_end_frame_image_url = match request.end_frame_image_media_token.as_ref() {
None => None,
Some(token) => match image_urls_by_token.get(token) {
Some(url) => Some(url.to_string()),
None => {
return Err(CommonWebError::BadInputWithSimpleMessage("Media for end frame not found.".to_string()));
}
}
};
insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string())
})?;
info!("Fal webhook URL: {}", server_state.fal.webhook_url);
let fal_result;
let generate_audio = request.generate_audio.unwrap_or(true);
if let Some(start_frame_url) = maybe_start_frame_image_url {
if let Some(end_frame_url) = maybe_end_frame_image_url {
info!("image-to-video case (start and end frame)");
let duration = match request.duration {
Some(Veo3p1FastMultiFunctionVideoGenDuration::FourSeconds) => EnqueueVeo3p1FastFirstLastFrameImageToVideoDurationSeconds::Four,
Some(Veo3p1FastMultiFunctionVideoGenDuration::SixSeconds) => EnqueueVeo3p1FastFirstLastFrameImageToVideoDurationSeconds::Six,
Some(Veo3p1FastMultiFunctionVideoGenDuration::EightSeconds) => EnqueueVeo3p1FastFirstLastFrameImageToVideoDurationSeconds::Eight,
None => EnqueueVeo3p1FastFirstLastFrameImageToVideoDurationSeconds::Six,
};
let aspect_ratio = match request.aspect_ratio {
Some(Veo3p1FastMultiFunctionVideoGenAspectRatio::Auto) => EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio::Auto,
Some(Veo3p1FastMultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio::SixteenByNine,
Some(Veo3p1FastMultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio::NineBySixteen,
None => EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio::SixteenByNine,
};
let resolution = match request.resolution {
Some(Veo3p1FastMultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueVeo3p1FastFirstLastFrameImageToVideoResolution::SevenTwentyP,
Some(Veo3p1FastMultiFunctionVideoGenResolution::TenEightyP) => EnqueueVeo3p1FastFirstLastFrameImageToVideoResolution::TenEightyP,
None => EnqueueVeo3p1FastFirstLastFrameImageToVideoResolution::TenEightyP,
};
let args = EnqueueVeo3p1FastFirstLastFrameImageToVideoArgs {
last_frame_url: start_frame_url,
first_frame_url: end_frame_url,
prompt: request.prompt.as_deref().unwrap_or("").to_string(),
duration: Some(duration),
aspect_ratio: Some(aspect_ratio),
resolution: Some(resolution),
generate_audio: Some(generate_audio),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
fal_result = enqueue_veo_3p1_fast_first_last_frame_image_to_video_webhook(args)
.await
.map_err(|err| {
warn!("Error calling veo_3p1_fast_first_last_frame_image_to_video_webhook: {:?}", err);
CommonWebError::ServerError
})?;
} else {
info!("image-to-video case (start frame only)");
let duration = match request.duration {
Some(Veo3p1FastMultiFunctionVideoGenDuration::FourSeconds) => EnqueueVeo3p1FastImageToVideoDurationSeconds::Four,
Some(Veo3p1FastMultiFunctionVideoGenDuration::SixSeconds) => EnqueueVeo3p1FastImageToVideoDurationSeconds::Six,
Some(Veo3p1FastMultiFunctionVideoGenDuration::EightSeconds) => EnqueueVeo3p1FastImageToVideoDurationSeconds::Eight,
None => EnqueueVeo3p1FastImageToVideoDurationSeconds::Six,
};
let aspect_ratio = match request.aspect_ratio {
Some(Veo3p1FastMultiFunctionVideoGenAspectRatio::Auto) => EnqueueVeo3p1FastImageToVideoAspectRatio::Auto,
Some(Veo3p1FastMultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueVeo3p1FastImageToVideoAspectRatio::SixteenByNine,
Some(Veo3p1FastMultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueVeo3p1FastImageToVideoAspectRatio::NineBySixteen,
None => EnqueueVeo3p1FastImageToVideoAspectRatio::SixteenByNine,
};
let resolution = match request.resolution {
Some(Veo3p1FastMultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueVeo3p1FastImageToVideoResolution::SevenTwentyP,
Some(Veo3p1FastMultiFunctionVideoGenResolution::TenEightyP) => EnqueueVeo3p1FastImageToVideoResolution::TenEightyP,
None => EnqueueVeo3p1FastImageToVideoResolution::TenEightyP,
};
let args = EnqueueVeo3p1FastImageToVideoArgs {
prompt: request.prompt.as_deref().unwrap_or("").to_string(),
image_url: start_frame_url,
duration: Some(duration),
aspect_ratio: Some(aspect_ratio),
resolution: Some(resolution),
generate_audio: Some(generate_audio),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
fal_result = enqueue_veo_3p1_fast_image_to_video_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_veo_3p1_fast_image_to_video_webhook: {:?}", err);
CommonWebError::ServerError
})?;
}
} else {
info!("text-to-video case");
let duration = match request.duration {
Some(Veo3p1FastMultiFunctionVideoGenDuration::FourSeconds) => EnqueueVeo3p1FastTextToVideoDurationSeconds::Four,
Some(Veo3p1FastMultiFunctionVideoGenDuration::SixSeconds) => EnqueueVeo3p1FastTextToVideoDurationSeconds::Six,
Some(Veo3p1FastMultiFunctionVideoGenDuration::EightSeconds) => EnqueueVeo3p1FastTextToVideoDurationSeconds::Eight,
None => EnqueueVeo3p1FastTextToVideoDurationSeconds::Six,
};
let aspect_ratio = match request.aspect_ratio {
Some(Veo3p1FastMultiFunctionVideoGenAspectRatio::Auto) => EnqueueVeo3p1FastTextToVideoAspectRatio::Auto,
Some(Veo3p1FastMultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueVeo3p1FastTextToVideoAspectRatio::SixteenByNine,
Some(Veo3p1FastMultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueVeo3p1FastTextToVideoAspectRatio::NineBySixteen,
None => EnqueueVeo3p1FastTextToVideoAspectRatio::SixteenByNine,
};
let resolution = match request.resolution {
Some(Veo3p1FastMultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueVeo3p1FastTextToVideoResolution::SevenTwentyP,
Some(Veo3p1FastMultiFunctionVideoGenResolution::TenEightyP) => EnqueueVeo3p1FastTextToVideoResolution::TenEightyP,
None => EnqueueVeo3p1FastTextToVideoResolution::TenEightyP,
};
let args = EnqueueVeo3p1FastTextToVideoArgs {
prompt: request.prompt.as_deref().unwrap_or("").to_string(),
negative_prompt: request.negative_prompt.clone(),
duration: Some(duration),
aspect_ratio: Some(aspect_ratio),
resolution: Some(resolution),
generate_audio: Some(generate_audio),
enhance_prompt: Some(true),
auto_fix: Some(true),
seed: None,
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
fal_result = enqueue_veo_3p1_fast_text_to_video_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_veo_3p1_fast_text_to_video_webhook: {:?}", err);
CommonWebError::ServerError
})?;
}
let external_job_id = fal_result.request_id
.ok_or_else(|| {
warn!("Fal request_id is None");
CommonWebError::ServerError
})?;
info!("Fal request_id: {}", external_job_id);
let ip_address = get_request_ip(&http_request);
let mut transaction = mysql_connection
.begin()
.await
.map_err(|err| {
error!("Error starting MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
// NB: Don't fail the job if the query fails.
let prompt_result = insert_prompt(InsertPromptArgs {
maybe_apriori_prompt_token: None,
prompt_type: PromptType::ArtcraftApp,
maybe_creator_user_token: maybe_user_session
.as_ref()
.map(|s| &s.user_token),
maybe_model_type: Some(ModelType::Veo3p1Fast),
maybe_generation_provider: Some(GenerationProvider::Artcraft),
maybe_positive_prompt: request.prompt.as_deref(),
maybe_negative_prompt: None,
maybe_other_args: None,
creator_ip_address: &ip_address,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let prompt_token = match prompt_result {
Ok(token) => Some(token),
Err(err) => {
warn!("Error inserting prompt: {:?}", err);
None // Don't fail the job if the prompt insertion fails.
}
};
if let Some(token) = prompt_token.as_ref() {
let mut context_items = Vec::with_capacity(2);
if let Some(media_token) = &request.start_frame_image_media_token {
context_items.push(PromptContextItem {
media_token: media_token.clone(),
context_semantic_type: PromptContextSemanticType::VidStartFrame,
});
}
if let Some(media_token) = &request.end_frame_image_media_token {
context_items.push(PromptContextItem {
media_token: media_token.clone(),
context_semantic_type: PromptContextSemanticType::VidEndFrame,
});
}
if !context_items.is_empty() {
let result = insert_batch_prompt_context_items(InsertBatchArgs {
prompt_token: token.clone(),
items: context_items,
transaction: &mut transaction,
}).await;
if let Err(err) = result {
// NB: Fail open.
warn!("Error inserting batch prompt context items: {:?}", err);
}
}
}
let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs {
uuid_idempotency_token: &request.uuid_idempotency_token,
maybe_external_third_party_id: &external_job_id,
fal_category: FalCategory::VideoGeneration,
maybe_inference_args: None,
maybe_prompt_token: prompt_token.as_ref(),
maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: Visibility::Public,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let job_token = match db_result {
Ok(token) => token,
Err(err) => {
warn!("Error inserting generic inference job for FAL queue: {:?}", err);
return Err(CommonWebError::ServerError);
}
};
let _r = transaction
.commit()
.await
.map_err(|err| {
error!("Error committing MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
Ok(Json(Veo3p1FastMultiFunctionVideoGenResponse {
success: true,
inference_job_token: job_token,
}))
}
@@ -0,0 +1,390 @@
use std::collections::{HashMap, HashSet};
use std::iter::FromIterator;
use std::sync::Arc;
use crate::http_server::common_responses::common_web_error::CommonWebError;
use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder;
use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test;
use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::state::server_state::ServerState;
use crate::util::lookup::lookup_image_urls_as_map::lookup_image_urls_as_map;
use actix_web::web::Json;
use actix_web::{web, HttpRequest};
use artcraft_api_defs::generate::video::multi_function::veo_3p1_multi_function_video_gen::{Veo3p1MultiFunctionVideoGenAspectRatio, Veo3p1MultiFunctionVideoGenDuration, Veo3p1MultiFunctionVideoGenRequest, Veo3p1MultiFunctionVideoGenResolution, Veo3p1MultiFunctionVideoGenResponse};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType;
use enums::by_table::prompts::prompt_type::PromptType;
use enums::common::generation_provider::GenerationProvider;
use enums::common::model_type::ModelType;
use enums::common::visibility::Visibility;
use fal_client::creds::open_ai_api_key::OpenAiApiKey;
use fal_client::requests::webhook::video::image::enqueue_veo_3p1_first_last_frame_image_to_video_webhook::{enqueue_veo_3p1_first_last_frame_image_to_video_webhook, EnqueueVeo3p1FirstLastFrameImageToVideoArgs, EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio, EnqueueVeo3p1FirstLastFrameImageToVideoDurationSeconds, EnqueueVeo3p1FirstLastFrameImageToVideoResolution};
use fal_client::requests::webhook::video::image::enqueue_veo_3p1_image_to_video_webhook::{enqueue_veo_3p1_image_to_video_webhook, EnqueueVeo3p1ImageToVideoArgs, EnqueueVeo3p1ImageToVideoAspectRatio, EnqueueVeo3p1ImageToVideoDurationSeconds, EnqueueVeo3p1ImageToVideoResolution};
use fal_client::requests::webhook::video::text::enqueue_veo_3p1_text_to_video_webhook::{enqueue_veo_3p1_text_to_video_webhook, EnqueueVeo3p1TextToVideoArgs, EnqueueVeo3p1TextToVideoAspectRatio, EnqueueVeo3p1TextToVideoDurationSeconds, EnqueueVeo3p1TextToVideoResolution};
use http_server_common::request::get_request_ip::get_request_ip;
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs;
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection};
use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem};
use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs};
use server_environment::ServerEnvironment;
use sqlx::pool::PoolConnection;
use sqlx::{Acquire, MySql};
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
/// Veo 3.1 Multi-Function (text and image to video)
#[utoipa::path(
post,
tag = "Generate Video (Multi-Function)",
path = "/v1/generate/video/multi_function/veo_3p1",
responses(
(status = 200, description = "Success", body = Veo3p1MultiFunctionVideoGenResponse),
),
params(
("request" = Veo3p1MultiFunctionVideoGenRequest, description = "Payload for Request"),
)
)]
pub async fn veo_3p1_multi_function_video_gen_handler(
http_request: HttpRequest,
request: Json<Veo3p1MultiFunctionVideoGenRequest>,
server_state: web::Data<Arc<ServerState>>
) -> Result<Json<Veo3p1MultiFunctionVideoGenResponse>, CommonWebError> {
payments_error_test(&request.prompt.as_deref().unwrap_or(""))?;
let mut mysql_connection = server_state.mysql_pool
.acquire()
.await?;
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_from_connection(&http_request, &mut mysql_connection)
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
CommonWebError::ServerError
})?;
let maybe_avt_token = server_state
.avt_cookie_manager
.get_avt_token_from_request(&http_request);
// TODO: Limit usage for new accounts. Billing, free credits metering, etc.
//let user_session = match maybe_user_session {
// Some(session) => session,
// None => {
// warn!("not logged in");
// return Err(CommonWebError::NotAuthorized);
// }
//};
if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
return Err(CommonWebError::BadInputWithSimpleMessage(reason));
}
let mut query_media_tokens = None;
if let Some(start_frame_token) = request.start_frame_image_media_token.as_ref() {
let mut tokens = Vec::new();
tokens.push(start_frame_token.to_owned());
if let Some(end_frame_token) = request.end_frame_image_media_token.as_ref() {
// NB: Masks are only relevant when this is an image editing exercise.
tokens.push(end_frame_token.to_owned());
}
query_media_tokens = Some(tokens);
}
let image_urls_by_token = match query_media_tokens {
None => HashMap::new(),
Some(media_tokens) => {
info!("Looking up image media tokens: {:?}", media_tokens);
lookup_image_urls_as_map(
&http_request,
&mut mysql_connection,
server_state.server_environment,
&media_tokens,
).await?
}
};
let maybe_start_frame_image_url = match request.start_frame_image_media_token.as_ref() {
None => None,
Some(token) => match image_urls_by_token.get(token) {
Some(url) => Some(url.to_string()),
None => {
return Err(CommonWebError::BadInputWithSimpleMessage("Media for start frame not found.".to_string()));
}
}
};
let maybe_end_frame_image_url = match request.end_frame_image_media_token.as_ref() {
None => None,
Some(token) => match image_urls_by_token.get(token) {
Some(url) => Some(url.to_string()),
None => {
return Err(CommonWebError::BadInputWithSimpleMessage("Media for end frame not found.".to_string()));
}
}
};
insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string())
})?;
info!("Fal webhook URL: {}", server_state.fal.webhook_url);
let fal_result;
let generate_audio = request.generate_audio.unwrap_or(true);
if let Some(start_frame_url) = maybe_start_frame_image_url {
if let Some(end_frame_url) = maybe_end_frame_image_url {
info!("image-to-video case (start and end frame)");
let duration = match request.duration {
Some(Veo3p1MultiFunctionVideoGenDuration::FourSeconds) => EnqueueVeo3p1FirstLastFrameImageToVideoDurationSeconds::Four,
Some(Veo3p1MultiFunctionVideoGenDuration::SixSeconds) => EnqueueVeo3p1FirstLastFrameImageToVideoDurationSeconds::Six,
Some(Veo3p1MultiFunctionVideoGenDuration::EightSeconds) => EnqueueVeo3p1FirstLastFrameImageToVideoDurationSeconds::Eight,
None => EnqueueVeo3p1FirstLastFrameImageToVideoDurationSeconds::Six,
};
let aspect_ratio = match request.aspect_ratio {
Some(Veo3p1MultiFunctionVideoGenAspectRatio::Auto) => EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio::Auto,
Some(Veo3p1MultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio::SixteenByNine,
Some(Veo3p1MultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio::NineBySixteen,
None => EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio::SixteenByNine,
};
let resolution = match request.resolution {
Some(Veo3p1MultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueVeo3p1FirstLastFrameImageToVideoResolution::SevenTwentyP,
Some(Veo3p1MultiFunctionVideoGenResolution::TenEightyP) => EnqueueVeo3p1FirstLastFrameImageToVideoResolution::TenEightyP,
None => EnqueueVeo3p1FirstLastFrameImageToVideoResolution::TenEightyP,
};
let args = EnqueueVeo3p1FirstLastFrameImageToVideoArgs {
last_frame_url: start_frame_url,
first_frame_url: end_frame_url,
prompt: request.prompt.as_deref().unwrap_or("").to_string(),
duration: Some(duration),
aspect_ratio: Some(aspect_ratio),
resolution: Some(resolution),
generate_audio: Some(generate_audio),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
fal_result = enqueue_veo_3p1_first_last_frame_image_to_video_webhook(args)
.await
.map_err(|err| {
warn!("Error calling veo_3p1_first_last_frame_image_to_video_webhook: {:?}", err);
CommonWebError::ServerError
})?;
} else {
info!("image-to-video case (start frame only)");
let duration = match request.duration {
Some(Veo3p1MultiFunctionVideoGenDuration::FourSeconds) => EnqueueVeo3p1ImageToVideoDurationSeconds::Four,
Some(Veo3p1MultiFunctionVideoGenDuration::SixSeconds) => EnqueueVeo3p1ImageToVideoDurationSeconds::Six,
Some(Veo3p1MultiFunctionVideoGenDuration::EightSeconds) => EnqueueVeo3p1ImageToVideoDurationSeconds::Eight,
None => EnqueueVeo3p1ImageToVideoDurationSeconds::Six,
};
let aspect_ratio = match request.aspect_ratio {
Some(Veo3p1MultiFunctionVideoGenAspectRatio::Auto) => EnqueueVeo3p1ImageToVideoAspectRatio::Auto,
Some(Veo3p1MultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueVeo3p1ImageToVideoAspectRatio::SixteenByNine,
Some(Veo3p1MultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueVeo3p1ImageToVideoAspectRatio::NineBySixteen,
None => EnqueueVeo3p1ImageToVideoAspectRatio::SixteenByNine,
};
let resolution = match request.resolution {
Some(Veo3p1MultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueVeo3p1ImageToVideoResolution::SevenTwentyP,
Some(Veo3p1MultiFunctionVideoGenResolution::TenEightyP) => EnqueueVeo3p1ImageToVideoResolution::TenEightyP,
None => EnqueueVeo3p1ImageToVideoResolution::TenEightyP,
};
let args = EnqueueVeo3p1ImageToVideoArgs {
prompt: request.prompt.as_deref().unwrap_or("").to_string(),
image_url: start_frame_url,
duration: Some(duration),
aspect_ratio: Some(aspect_ratio),
resolution: Some(resolution),
generate_audio: Some(generate_audio),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
fal_result = enqueue_veo_3p1_image_to_video_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_veo_3p1_image_to_video_webhook: {:?}", err);
CommonWebError::ServerError
})?;
}
} else {
info!("text-to-video case");
let duration = match request.duration {
Some(Veo3p1MultiFunctionVideoGenDuration::FourSeconds) => EnqueueVeo3p1TextToVideoDurationSeconds::Four,
Some(Veo3p1MultiFunctionVideoGenDuration::SixSeconds) => EnqueueVeo3p1TextToVideoDurationSeconds::Six,
Some(Veo3p1MultiFunctionVideoGenDuration::EightSeconds) => EnqueueVeo3p1TextToVideoDurationSeconds::Eight,
None => EnqueueVeo3p1TextToVideoDurationSeconds::Six,
};
let aspect_ratio = match request.aspect_ratio {
Some(Veo3p1MultiFunctionVideoGenAspectRatio::Auto) => EnqueueVeo3p1TextToVideoAspectRatio::Auto,
Some(Veo3p1MultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueVeo3p1TextToVideoAspectRatio::SixteenByNine,
Some(Veo3p1MultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueVeo3p1TextToVideoAspectRatio::NineBySixteen,
None => EnqueueVeo3p1TextToVideoAspectRatio::SixteenByNine,
};
let resolution = match request.resolution {
Some(Veo3p1MultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueVeo3p1TextToVideoResolution::SevenTwentyP,
Some(Veo3p1MultiFunctionVideoGenResolution::TenEightyP) => EnqueueVeo3p1TextToVideoResolution::TenEightyP,
None => EnqueueVeo3p1TextToVideoResolution::TenEightyP,
};
let args = EnqueueVeo3p1TextToVideoArgs {
prompt: request.prompt.as_deref().unwrap_or("").to_string(),
negative_prompt: request.negative_prompt.clone(),
duration: Some(duration),
aspect_ratio: Some(aspect_ratio),
resolution: Some(resolution),
generate_audio: Some(generate_audio),
enhance_prompt: Some(true),
auto_fix: Some(true),
seed: None,
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
fal_result = enqueue_veo_3p1_text_to_video_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_veo_3p1_text_to_video_webhook: {:?}", err);
CommonWebError::ServerError
})?;
}
let external_job_id = fal_result.request_id
.ok_or_else(|| {
warn!("Fal request_id is None");
CommonWebError::ServerError
})?;
info!("Fal request_id: {}", external_job_id);
let ip_address = get_request_ip(&http_request);
let mut transaction = mysql_connection
.begin()
.await
.map_err(|err| {
error!("Error starting MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
// NB: Don't fail the job if the query fails.
let prompt_result = insert_prompt(InsertPromptArgs {
maybe_apriori_prompt_token: None,
prompt_type: PromptType::ArtcraftApp,
maybe_creator_user_token: maybe_user_session
.as_ref()
.map(|s| &s.user_token),
maybe_model_type: Some(ModelType::Veo3p1),
maybe_generation_provider: Some(GenerationProvider::Artcraft),
maybe_positive_prompt: request.prompt.as_deref(),
maybe_negative_prompt: None,
maybe_other_args: None,
creator_ip_address: &ip_address,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let prompt_token = match prompt_result {
Ok(token) => Some(token),
Err(err) => {
warn!("Error inserting prompt: {:?}", err);
None // Don't fail the job if the prompt insertion fails.
}
};
if let Some(token) = prompt_token.as_ref() {
let mut context_items = Vec::with_capacity(2);
if let Some(media_token) = &request.start_frame_image_media_token {
context_items.push(PromptContextItem {
media_token: media_token.clone(),
context_semantic_type: PromptContextSemanticType::VidStartFrame,
});
}
if let Some(media_token) = &request.end_frame_image_media_token {
context_items.push(PromptContextItem {
media_token: media_token.clone(),
context_semantic_type: PromptContextSemanticType::VidEndFrame,
});
}
if !context_items.is_empty() {
let result = insert_batch_prompt_context_items(InsertBatchArgs {
prompt_token: token.clone(),
items: context_items,
transaction: &mut transaction,
}).await;
if let Err(err) = result {
// NB: Fail open.
warn!("Error inserting batch prompt context items: {:?}", err);
}
}
}
let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs {
uuid_idempotency_token: &request.uuid_idempotency_token,
maybe_external_third_party_id: &external_job_id,
fal_category: FalCategory::VideoGeneration,
maybe_inference_args: None,
maybe_prompt_token: prompt_token.as_ref(),
maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: Visibility::Public,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let job_token = match db_result {
Ok(token) => token,
Err(err) => {
warn!("Error inserting generic inference job for FAL queue: {:?}", err);
return Err(CommonWebError::ServerError);
}
};
let _r = transaction
.commit()
.await
.map_err(|err| {
error!("Error committing MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
Ok(Json(Veo3p1MultiFunctionVideoGenResponse {
success: true,
inference_job_token: job_token,
}))
}
@@ -25,6 +25,12 @@ use crate::http_server::endpoints::generate::video::generate_seedance_1_0_pro_im
use crate::http_server::endpoints::generate::video::generate_veo_2_image_to_video_handler::generate_veo_2_image_to_video_handler;
use crate::http_server::endpoints::generate::video::generate_veo_3_fast_image_to_video_handler::generate_veo_3_fast_image_to_video_handler;
use crate::http_server::endpoints::generate::video::generate_veo_3_image_to_video_handler::generate_veo_3_image_to_video_handler;
use crate::http_server::endpoints::generate::video::multi_function::kling_2p5_turbo_pro_multi_function_video_gen_handler::kling_2p5_turbo_pro_multi_function_video_gen_handler;
use crate::http_server::endpoints::generate::video::multi_function::kling_2p6_pro_multi_function_video_gen_handler::kling_2p6_pro_multi_function_video_gen_handler;
use crate::http_server::endpoints::generate::video::multi_function::sora_2_multi_function_video_gen_handler::sora_2_multi_function_video_gen_handler;
use crate::http_server::endpoints::generate::video::multi_function::sora_2_pro_multi_function_video_gen_handler::sora_2_pro_multi_function_video_gen_handler;
use crate::http_server::endpoints::generate::video::multi_function::veo_3p1_fast_multi_function_video_gen_handler::veo_3p1_fast_multi_function_video_gen_handler;
use crate::http_server::endpoints::generate::video::multi_function::veo_3p1_multi_function_video_gen_handler::veo_3p1_multi_function_video_gen_handler;
use actix_http::body::MessageBody;
use actix_service::ServiceFactory;
use actix_web::dev::{ServiceRequest, ServiceResponse};
@@ -119,6 +125,32 @@ where
)
)
.service(web::scope("/video")
.service(web::scope("/multi_function")
.service(web::resource("/kling_2p5_turbo_pro")
.route(web::post().to(kling_2p5_turbo_pro_multi_function_video_gen_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
.service(web::resource("/kling_2p6_pro")
.route(web::post().to(kling_2p6_pro_multi_function_video_gen_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
.service(web::resource("/sora_2")
.route(web::post().to(sora_2_multi_function_video_gen_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
.service(web::resource("/sora_2_pro")
.route(web::post().to(sora_2_pro_multi_function_video_gen_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
.service(web::resource("/veo_3p1")
.route(web::post().to(veo_3p1_multi_function_video_gen_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
.service(web::resource("/veo_3p1_fast")
.route(web::post().to(veo_3p1_fast_multi_function_video_gen_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
)
.service(web::resource("/kling_1.6_pro_image_to_video")
.route(web::post().to(generate_kling_1_6_pro_video_handler))
.route(web::head().to(|| HttpResponse::Ok()))