Compare commits

...
Author SHA1 Message Date
Kasisnu ca7eb571e7 scaffold for bvh -> workflow job 2024-01-26 15:21:22 +05:30
10 changed files with 405 additions and 1 deletions
@@ -41,6 +41,10 @@ pub enum InferenceCategory {
/// A job that turns "FBX" game engine files into "GLTF" files (Bevy-compatible).
#[serde(rename = "convert_fbx_gltf")]
ConvertFbxToGltf,
#[serde(rename = "convert_bvh_to_workflow")]
ConvertBvhToWorkflow,
}
// TODO(bt, 2022-12-21): This desperately needs MySQL integration tests!
@@ -59,6 +63,7 @@ impl InferenceCategory {
Self::Mocap => "mocap",
Self::Workflow => "workflow",
Self::ConvertFbxToGltf => "convert_fbx_gltf",
Self::ConvertBvhToWorkflow => "convert_bvh_to_workflow",
}
}
@@ -72,6 +77,7 @@ impl InferenceCategory {
"mocap" => Ok(Self::Mocap),
"workflow" => Ok(Self::Workflow),
"convert_fbx_gltf" => Ok(Self::ConvertFbxToGltf),
"convert_bvh_to_workflow" => Ok(Self::ConvertBvhToWorkflow),
_ => Err(format!("invalid value: {:?}", value)),
}
}
@@ -88,6 +94,7 @@ impl InferenceCategory {
Self::Mocap,
Self::Workflow,
Self::ConvertFbxToGltf,
Self::ConvertBvhToWorkflow,
])
}
}
@@ -110,6 +117,7 @@ mod tests {
assert_serialization(InferenceCategory::Mocap, "mocap");
assert_serialization(InferenceCategory::Workflow, "workflow");
assert_serialization(InferenceCategory::ConvertFbxToGltf, "convert_fbx_gltf");
assert_serialization(InferenceCategory::ConvertBvhToWorkflow, "convert_bvh_to_workflow");
}
#[test]
@@ -122,6 +130,7 @@ mod tests {
assert_eq!(InferenceCategory::Mocap.to_str(), "mocap");
assert_eq!(InferenceCategory::Workflow.to_str(), "workflow");
assert_eq!(InferenceCategory::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
assert_eq!(InferenceCategory::ConvertBvhToWorkflow.to_str(), "convert_bvh_to_workflow");
}
#[test]
@@ -134,6 +143,7 @@ mod tests {
assert_eq!(InferenceCategory::from_str("mocap").unwrap(), InferenceCategory::Mocap);
assert_eq!(InferenceCategory::from_str("workflow").unwrap(), InferenceCategory::Workflow);
assert_eq!(InferenceCategory::from_str("convert_fbx_gltf").unwrap(), InferenceCategory::ConvertFbxToGltf);
assert_eq!(InferenceCategory::from_str("convert_bvh_to_workflow").unwrap(), InferenceCategory::ConvertBvhToWorkflow);
}
#[test]
@@ -149,6 +159,7 @@ mod tests {
assert_eq!(variants.pop_first(), Some(InferenceCategory::Mocap));
assert_eq!(variants.pop_first(), Some(InferenceCategory::Workflow));
assert_eq!(variants.pop_first(), Some(InferenceCategory::ConvertFbxToGltf));
assert_eq!(variants.pop_first(), Some(InferenceCategory::ConvertBvhToWorkflow));
assert_eq!(variants.pop_first(), None);
// Generated check
@@ -41,6 +41,8 @@ pub enum InferenceModelType {
ComfyUi,
#[serde(rename = "styletts2")]
StyleTTS2,
#[serde(rename = "bvh_to_workflow")]
BvhToWorkflow
}
// TODO(bt, 2022-12-21): This desperately needs MySQL integration tests!
@@ -62,6 +64,7 @@ impl InferenceModelType {
Self::MocapNet => "mocap_net",
Self::StyleTTS2 => "styletts2",
Self::ComfyUi => "comfy_ui",
Self::BvhToWorkflow => "bvh_to_workflow",
}
}
@@ -78,6 +81,7 @@ impl InferenceModelType {
"mocap_net" => Ok(Self::MocapNet),
"styletts2" => Ok(Self::StyleTTS2),
"comfy_ui" => Ok(Self::ComfyUi),
"bvh_to_workflow" => Ok(Self::BvhToWorkflow),
_ => Err(format!("invalid value: {:?}", value)),
}
}
@@ -97,6 +101,7 @@ impl InferenceModelType {
InferenceModelType::MocapNet,
InferenceModelType::StyleTTS2,
InferenceModelType::ComfyUi,
InferenceModelType::BvhToWorkflow,
])
}
}
@@ -122,6 +127,7 @@ mod tests {
assert_serialization(InferenceModelType::MocapNet, "mocap_net");
assert_serialization(InferenceModelType::ComfyUi, "comfy_ui");
assert_serialization(InferenceModelType::StyleTTS2, "styletts2");
assert_serialization(InferenceModelType::BvhToWorkflow, "bvh_to_workflow");
}
#[test]
@@ -137,6 +143,7 @@ mod tests {
assert_eq!(InferenceModelType::MocapNet.to_str(), "mocap_net");
assert_eq!(InferenceModelType::StyleTTS2.to_str(), "styletts2");
assert_eq!(InferenceModelType::ComfyUi.to_str(), "comfy_ui");
assert_eq!(InferenceModelType::BvhToWorkflow.to_str(), "bvh_to_workflow");
}
#[test]
@@ -152,13 +159,14 @@ mod tests {
assert_eq!(InferenceModelType::from_str("mocap_net").unwrap(), InferenceModelType::MocapNet);
assert_eq!(InferenceModelType::from_str("styletts2").unwrap(), InferenceModelType::StyleTTS2);
assert_eq!(InferenceModelType::from_str("comfy_ui").unwrap(), InferenceModelType::ComfyUi);
assert_eq!(InferenceModelType::from_str("bvh_to_workflow").unwrap(), InferenceModelType::BvhToWorkflow);
}
#[test]
fn all_variants() {
// Static check
let mut variants = InferenceModelType::all_variants();
assert_eq!(variants.len(), 11);
assert_eq!(variants.len(), 12);
assert_eq!(variants.pop_first(), Some(InferenceModelType::RvcV2));
assert_eq!(variants.pop_first(), Some(InferenceModelType::SadTalker));
assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc));
@@ -170,6 +178,7 @@ mod tests {
assert_eq!(variants.pop_first(), Some(InferenceModelType::MocapNet));
assert_eq!(variants.pop_first(), Some(InferenceModelType::ComfyUi));
assert_eq!(variants.pop_first(), Some(InferenceModelType::StyleTTS2));
assert_eq!(variants.pop_first(), Some(InferenceModelType::BvhToWorkflow));
assert_eq!(variants.pop_first(), None);
// Generated check
@@ -58,6 +58,10 @@ pub enum InferenceCategoryAbbreviated {
#[serde(rename = "fg")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
#[serde(alias = "convert_fbx_gltf")]
ConvertFbxToGltf,
#[serde(rename = "bw")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
#[serde(alias = "convert_bvh_to_workflow")]
ConvertBvhToWorkflow,
}
#[derive(Serialize, Deserialize, Clone, Copy, Debug)]
@@ -143,6 +147,7 @@ impl InferenceCategoryAbbreviated {
InferenceCategory::Mocap => Self::Mocap,
InferenceCategory::Workflow => Self::Workflow,
InferenceCategory::ConvertFbxToGltf => Self::ConvertFbxToGltf,
InferenceCategory::ConvertBvhToWorkflow => Self::ConvertBvhToWorkflow,
}
}
@@ -156,6 +161,7 @@ impl InferenceCategoryAbbreviated {
Self::Mocap => InferenceCategory::Mocap,
Self::Workflow => InferenceCategory::Workflow,
Self::ConvertFbxToGltf => InferenceCategory::ConvertFbxToGltf,
Self::ConvertBvhToWorkflow => InferenceCategory::ConvertBvhToWorkflow,
}
}
}
@@ -183,6 +183,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI
InferenceCategory::Mocap => Some("Mocap"),
InferenceCategory::Workflow => Some("Workflow"),
InferenceCategory::ConvertFbxToGltf => Some("FBX to GLTF"),
InferenceCategory::ConvertBvhToWorkflow => Some("BVH to Workflow"),
};
// NB: A bit of a hack. We store TTS results with a full path.
@@ -196,6 +197,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI
InferenceCategory::Mocap => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::Workflow => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::ConvertFbxToGltf => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::ConvertBvhToWorkflow => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
};
// NB: We've moved voice conversion out of their own table and into media_files
@@ -13,6 +13,7 @@ use redis_common::redis_keys::RedisKeys;
use crate::job::job_loop::determine_dependency_status::determine_dependency_status;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_loop::process_single_job_success_case::ProcessSingleJobSuccessCase;
use crate::job::job_types::bevy_to_workflow::process_single_bevy_to_workflow_conversion_job::process_single_bevy_to_workflow_conversion_job;
use crate::job::job_types::format_conversion::process_single_format_conversion_job::{FormatConversionType, process_single_format_conversion_job};
use crate::job::job_types::image_generation::process_single_ig_job::process_single_ig_job;
use crate::job::job_types::lipsync::process_single_lipsync_job::process_single_lipsync_job;
@@ -202,6 +203,9 @@ async fn do_process_single_job(
InferenceCategory::ConvertFbxToGltf => {
process_single_format_conversion_job(job_dependencies, job, FormatConversionType::FbxToGltf).await?
}
InferenceCategory::ConvertBvhToWorkflow => {
process_single_bevy_to_workflow_conversion_job(job_dependencies, job).await?
}
};
let maybe_entity_type = job_success_result.maybe_result_entity
@@ -0,0 +1 @@
pub mod process_job;
@@ -0,0 +1,318 @@
use std::time::Instant;
use anyhow::anyhow;
use log::info;
use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::get_media_file_for_inference::MediaFileForInference;
use crate::job::job_loop::job_success_result::JobSuccessResult;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job_dependencies::JobDependencies;
use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
const BUCKET_FILE_EXTENSION : &str = ".gltf";
pub struct BvhToWorkflowJobArgs<'a> {
pub job_dependencies: &'a JobDependencies,
pub job: &'a AvailableInferenceJob,
pub media_file: &'a MediaFileForInference,
}
pub async fn process_job(args: BvhToWorkflowJobArgs<'_>) -> Result<JobSuccessResult, ProcessSingleJobError> {
let job = args.job;
let media_file = args.media_file;
let mut job_progress_reporter = args.job_dependencies
.clients
.job_progress_reporter
.new_generic_inference(job.inference_job_token.as_str())
.map_err(|e| ProcessSingleJobError::Other(anyhow!(e)))?;
// ==================== TEMP DIR ==================== //
let work_temp_dir = format!("temp_file_convert_{}", job.id.0);
// NB: TempDir exists until it goes out of scope, at which point it should delete from filesystem.
let work_temp_dir = args.job_dependencies
.fs
.scoped_temp_dir_creator_for_work
.new_tempdir(&work_temp_dir)
.map_err(|e| ProcessSingleJobError::from_io_error(e))?;
// ==================== DOWNLOAD MEDIA FILE ==================== //
info!("Download media for conversion...");
//TODO(KS): update to match correct paths & extensions
let original_media_upload_fs_path = {
let original_media_upload_fs_path = work_temp_dir.path().join("original.fbx");
let media_upload_bucket_path =
MediaUploadOriginalFilePath::from_object_hash(&media_file.public_bucket_directory_hash);
let bucket_object_path = media_upload_bucket_path.to_full_object_pathbuf();
info!("Downloading media to bucket path: {:?}", &bucket_object_path);
maybe_download_file_from_bucket(MaybeDownloadArgs {
name_or_description_of_file: "media upload (original file)",
final_filesystem_file_path: &original_media_upload_fs_path,
bucket_object_path: &bucket_object_path,
bucket_client: &args.job_dependencies.buckets.public_bucket_client,
job_progress_reporter: &mut job_progress_reporter,
job_progress_update_description: "downloading",
job_id: job.id.0,
scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
maybe_existing_file_minimum_size_required: None,
}).await?;
original_media_upload_fs_path
};
// TODO: No duration. Replace this.
let inference_duration = Instant::now().duration_since(Instant::now());
/* // TODO
// ==================== SETUP FOR CONVERSION ==================== //
job_progress_reporter.log_status("running conversion")
.map_err(|e| ProcessSingleJobError::Other(e))?;
//let config_path = PathBuf::from("/models/voice_conversion/so-vits-svc/example_config.json"); // TODO: This could be variable.
let input_wav_path = original_media_upload_fs_path;
let output_audio_fs_path = work_temp_dir.path().join("output.wav");
let stderr_output_file = work_temp_dir.path().join("stderr.txt");
info!("Running VC inference...");
info!("Expected output audio filename: {:?}", &output_audio_fs_path);
// TODO: Limit output length for premium.
let maybe_args = job.maybe_inference_args
.as_ref()
.map(|args| args.args.as_ref())
.flatten();
// If not specified by the user, turn off auto prediction. It sounds awful.
let auto_predict_f0 = maybe_args
.map(|args| match args {
PolymorphicInferenceArgs::Tts { .. } => None,
PolymorphicInferenceArgs::La(_) => None,
PolymorphicInferenceArgs::Vc { auto_predict_f0, .. } => *auto_predict_f0,
PolymorphicInferenceArgs::Rr(_) => None,
PolymorphicInferenceArgs::Ig(_) => None,
PolymorphicInferenceArgs::Mc(_) => None,
PolymorphicInferenceArgs::Cu(_) => None,
})
.flatten()
.unwrap_or(false);
let maybe_transpose = maybe_args
.map(|args| match args {
PolymorphicInferenceArgs::Tts { .. } => None,
PolymorphicInferenceArgs::La(_) => None,
PolymorphicInferenceArgs::Vc { transpose, .. } => *transpose,
PolymorphicInferenceArgs::Rr(_) => None,
PolymorphicInferenceArgs::Ig(_) => None,
PolymorphicInferenceArgs::Mc(_) => None,
PolymorphicInferenceArgs::Cu(_) => None,
})
.flatten();
// ==================== RUN INFERENCE SCRIPT ==================== //
let inference_start_time = Instant::now();
let command_exit_status = model_dependencies
.inference_command
.execute_inference(InferenceArgs {
model_path: &so_vits_svc_fs_path,
input_path: &input_wav_path,
output_path: &output_audio_fs_path,
stderr_output_file: &stderr_output_file,
maybe_config_path: None,
device: Device::Cuda,
auto_predict_f0,
maybe_transpose,
});
let inference_duration = Instant::now().duration_since(inference_start_time);
info!("Inference took duration to complete: {:?}", &inference_duration);
if !command_exit_status.is_success() {
error!("Inference failed: {:?}", command_exit_status);
let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", command_exit_status));
if let Ok(contents) = read_to_string(&stderr_output_file) {
warn!("Captured stderr output: {}", contents);
//match categorize_error(&contents) {
// Some(ProcessSingleJobError::FaceDetectionFailure) => {
// warn!("Face not detected in source image");
// error = ProcessSingleJobError::FaceDetectionFailure;
// }
// _ => {}
//}
}
safe_delete_temp_file(&input_wav_path);
safe_delete_temp_file(&output_audio_fs_path);
safe_delete_temp_directory(&work_temp_dir);
return Err(error);
}
// ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== //
info!("Checking that output files exist...");
check_file_exists(&output_audio_fs_path).map_err(|e| ProcessSingleJobError::Other(e))?;
info!("Interrogating result file properties...");
let file_size_bytes = file_size(&output_audio_fs_path)
.map_err(|err| ProcessSingleJobError::Other(err))?;
let maybe_mimetype = get_mimetype_for_file(&output_audio_fs_path)
.map_err(|err| ProcessSingleJobError::from_io_error(err))?
.map(|mime| mime.to_string());
let audio_info = decode_basic_audio_file_info(&output_audio_fs_path, maybe_mimetype.as_deref(), None)
.map_err(|err| ProcessSingleJobError::Other(err))?;
if audio_info.required_full_decode {
warn!("Required a full decode of the output data to get duration! That's inefficient!");
}
info!("Calculating sha256...");
let file_checksum = sha256_hash_file(&output_audio_fs_path)
.map_err(|err| {
ProcessSingleJobError::Other(anyhow!("Error hashing file: {:?}", err))
})?;
let file_metadata = FileMetadata {
duration_millis: audio_info.duration_millis,
mimetype: maybe_mimetype.clone(),
file_size_bytes,
};
// ==================== UPLOAD AUDIO TO BUCKET ==================== //
job_progress_reporter.log_status("uploading result")
.map_err(|e| ProcessSingleJobError::Other(e))?;
// let result_bucket_location = VoiceConversionResultOriginalFilePath::generate_new();
//
// let result_bucket_object_pathbuf = result_bucket_location.to_full_object_pathbuf();
//
// info!("Audio destination bucket path: {:?}", &result_bucket_object_pathbuf);
let result_bucket_location = MediaFileBucketPath::generate_new(
Some(BUCKET_FILE_PREFIX),
Some(BUCKET_FILE_EXTENSION));
let result_bucket_object_pathbuf = result_bucket_location.to_full_object_pathbuf();
info!("Audio destination bucket path: {:?}", &result_bucket_object_pathbuf);
info!("Uploading audio media file...");
args.job_dependencies.buckets.public_bucket_client.upload_filename_with_content_type(
&result_bucket_object_pathbuf,
&output_audio_fs_path,
"audio/wav")
.await
.map_err(|e| ProcessSingleJobError::Other(e))?;
safe_delete_temp_file(&output_audio_fs_path);
// ==================== DELETE DOWNLOADED FILE ==================== //
// NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk...
safe_delete_temp_directory(&work_temp_dir);
// ==================== SAVE RECORDS ==================== //
info!("Saving vc inference record...");
// let (inference_result_token, id) = insert_voice_conversion_result(InsertArgs {
// pool: &args.job_dependencies.mysql_pool,
// job: &job,
// public_bucket_hash: result_bucket_location.get_object_hash(),
// file_size_bytes: file_metadata.file_size_bytes,
// duration_millis: file_metadata.duration_millis.unwrap_or(0),
// is_on_prem: args.job_dependencies.container.is_on_prem,
// worker_hostname: &args.job_dependencies.container.hostname,
// worker_cluster: &args.job_dependencies.container.cluster_name,
// is_debug_worker: args.job_dependencies.worker_details.is_debug_worker
// })
// .await
// .map_err(|e| ProcessSingleJobError::Other(e))?;
let (inference_result_token, id) = insert_media_file_from_voice_conversion(InsertMediaFileArgs {
pool: &args.job_dependencies.db.mysql_pool,
job: &job,
voice_conversion_type: VoiceConversionModelType::SoVitsSvc,
maybe_mime_type: maybe_mimetype.as_deref(),
file_size_bytes: file_metadata.file_size_bytes,
duration_millis: file_metadata.duration_millis.unwrap_or(0),
sha256_checksum: &file_checksum,
public_bucket_directory_hash: result_bucket_location.get_object_hash(),
maybe_public_bucket_prefix: Some(BUCKET_FILE_PREFIX),
maybe_public_bucket_extension: Some(BUCKET_FILE_EXTENSION),
is_on_prem: args.job_dependencies.job.info.container.is_on_prem,
worker_hostname: &args.job_dependencies.job.info.container.hostname,
worker_cluster: &args.job_dependencies.job.info.container.cluster_name,
})
.await
.map_err(|e| ProcessSingleJobError::Other(e))?;
info!("VC Done.");
// TODO: Update upstream to be strongly typed
let maybe_user_token = job.maybe_creator_user_token.as_deref()
.map(|token| UserToken::new_from_str(token));
args.job_dependencies.clients.firehose_publisher.vc_inference_finished(
maybe_user_token.as_ref(),
&job.inference_job_token,
inference_result_token.as_str())
.await
.map_err(|e| {
error!("error publishing event: {:?}", e);
ProcessSingleJobError::Other(anyhow!("error publishing event"))
})?;
job_progress_reporter.log_status("done")
.map_err(|e| ProcessSingleJobError::Other(e))?;
info!("Job {:?} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}, Result Token: {}",
job.id, id, &inference_result_token);
*/
Ok(JobSuccessResult {
//maybe_result_entity: Some(ResultEntity {
// entity_type: InferenceResultType::MediaFile,
// entity_token: inference_result_token.to_string(),
//}),
maybe_result_entity: None,
inference_duration,
})
}
#[derive(Deserialize, Default)]
struct FileMetadata {
pub duration_millis: Option<u64>,
pub mimetype: Option<String>,
pub file_size_bytes: u64,
}
@@ -0,0 +1,2 @@
pub mod bvh_to_workflow;
pub mod process_single_bevy_to_workflow_conversion_job;
@@ -0,0 +1,50 @@
use anyhow::anyhow;
use log::error;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::get_media_file_for_inference::get_media_file_for_inference;
use tokens::tokens::media_files::MediaFileToken;
use crate::job::job_loop::job_success_result::JobSuccessResult;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::bevy_to_workflow::bvh_to_workflow;
use crate::job::job_types::bevy_to_workflow::bvh_to_workflow::process_job::BvhToWorkflowJobArgs;
use crate::job_dependencies::JobDependencies;
pub async fn process_single_bevy_to_workflow_conversion_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result<JobSuccessResult, ProcessSingleJobError> {
let maybe_media_file_token = job.maybe_input_source_token
.as_deref()
.map(|token| MediaFileToken::new_from_str(token));
let media_file_token = match maybe_media_file_token {
None => return Err(ProcessSingleJobError::Other(anyhow!("no associated media file for format conversion job: {:?}", job.inference_job_token))),
Some(token) => token,
};
let maybe_media_file_result =
get_media_file_for_inference(&media_file_token, &job_dependencies.db.mysql_pool).await;
let media_file = match maybe_media_file_result {
Ok(Some(media_file)) => media_file,
Ok(None) => {
error!("no media file record found for token: {:?}", media_file_token);
return Err(ProcessSingleJobError::Other(anyhow!("no media file record found for token: {:?}", media_file_token)));
}
Err(err) => {
error!("error fetching media file record from db: {:?}", err);
return Err(ProcessSingleJobError::Other(err));
}
};
let job_success_result = bvh_to_workflow::process_job::process_job(BvhToWorkflowJobArgs {
job_dependencies,
job,
media_file: &media_file,
}).await?;
Ok(job_success_result)
}
@@ -6,3 +6,4 @@ pub mod image_generation;
pub(crate) mod mocap;
pub(crate) mod workflow;
pub mod format_conversion;
pub mod bevy_to_workflow;