comfy: pass audio for lipsyncing

crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/video_style_transfer/process_job.rs

@@ -50,6 +50,7 @@ use crate::job::job_types::workflow::comfy_ui::video_style_transfer::steps::down
 use crate::job::job_types::workflow::comfy_ui::video_style_transfer::steps::download_input_video::{download_input_video, DownloadInputVideoArgs};
 use crate::job::job_types::workflow::comfy_ui::video_style_transfer::steps::post_process_add_watermark::{post_process_add_watermark, PostProcessAddWatermarkArgs};
 use crate::job::job_types::workflow::comfy_ui::video_style_transfer::steps::post_process_restore_audio::{post_process_restore_audio, PostProcessRestoreVideoArgs};
+use crate::job::job_types::workflow::comfy_ui::video_style_transfer::steps::preprocess_save_audio::{preprocess_save_audio, ProcessSaveAudioArgs};
 use crate::job::job_types::workflow::comfy_ui::video_style_transfer::steps::preprocess_trim_and_resample_video::{preprocess_trim_and_resample_video, ProcessTrimAndResampleVideoArgs};
 use crate::job::job_types::workflow::comfy_ui::video_style_transfer::steps::validate_and_save_results::{SaveResultsArgs, validate_and_save_results};
 use crate::job::job_types::workflow::comfy_ui::video_style_transfer::video_paths::VideoPaths;
@@ -260,6 +261,18 @@ pub async fn process_job(args: ComfyProcessJobArgs<'_>) -> Result<JobSuccessResu
         videos: &videos,
     })?;
 
+    // ========================= PREPROCESS AUDIO ======================== //
+
+    let mut lipsync_enabled = comfy_args.lipsync_enabled.unwrap_or(false);
+    if let Err(err) = preprocess_save_audio(ProcessSaveAudioArgs {
+        comfy_deps: model_dependencies,
+        videos: &mut videos,
+    }) {
+        error!("Audio extraction failed: {:?}", err);
+
+        lipsync_enabled = false;
+    }
+
     // ========================= CREATE OUTPUT DIR ======================== //
 
     // make outputs dir if not exist
@@ -318,7 +331,7 @@ pub async fn process_job(args: ComfyProcessJobArgs<'_>) -> Result<JobSuccessResu
             inference_details,
             face_detailer_enabled: comfy_args.use_face_detailer.unwrap_or(false),
             upscaler_enabled: comfy_args.use_upscaler.unwrap_or(false),
-            lipsync_enabled: comfy_args.lipsync_enabled.unwrap_or(false),
+            lipsync_enabled,
             disable_lcm: comfy_args.disable_lcm.unwrap_or(false),
             use_cinematic: comfy_args.use_cinematic.unwrap_or(false),
             maybe_strength: comfy_args.strength,
@@ -372,6 +385,7 @@ pub async fn process_job(args: ComfyProcessJobArgs<'_>) -> Result<JobSuccessResu
 
         safe_delete_temp_file(&videos.comfy_input_video_path);
         safe_delete_temp_file(&videos.trimmed_resampled_video_path);
+        safe_delete_temp_file(&videos.trimmed_audio_path);
         safe_delete_temp_file(&videos.original_video_path);
 
         let output_dir = root_comfy_path.join("output");

crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/video_style_transfer/steps/mod.rs

@@ -5,3 +5,4 @@ pub mod post_process_add_watermark;
 pub mod post_process_restore_audio;
 pub mod preprocess_trim_and_resample_video;
 pub mod validate_and_save_results;
+pub mod preprocess_save_audio;

crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/video_style_transfer/steps/preprocess_save_audio.rs

@@ -0,0 +1,42 @@
+use anyhow::anyhow;
+use log::{error, info};
+
+use filesys::check_file_exists::check_file_exists;
+use mysql_queries::payloads::generic_inference_args::workflow_payload::WorkflowArgs;
+use subprocess_common::command_runner::command_runner_args::{RunAsSubprocessArgs, StreamRedirection};
+
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::job::job_types::workflow::comfy_ui::comfy_ui_dependencies::ComfyDependencies;
+use crate::job::job_types::workflow::comfy_ui::video_style_transfer::video_paths::VideoPaths;
+use crate::util::common_commands::ffmpeg_audio_replace_args::FfmpegAudioReplaceArgs;
+use crate::util::common_commands::ffmpeg_extract_audio_args::FfmpegExtractAudioArgs;
+
+pub struct ProcessSaveAudioArgs<'a> {
+    pub comfy_deps: &'a ComfyDependencies,
+    pub videos: &'a mut VideoPaths,
+}
+
+pub fn preprocess_save_audio(
+    args: ProcessSaveAudioArgs<'_>
+) -> Result<(), ProcessSingleJobError> {
+    info!("Extracting audio...");
+
+
+    let command_exit_status = args
+        .comfy_deps
+        .ffmpeg_command_runner
+        .run_with_subprocess(RunAsSubprocessArgs {
+            args: Box::new(&FfmpegExtractAudioArgs {
+                input_video_file: &args.videos.trimmed_resampled_video_path,
+                output_file: &args.videos.trimmed_audio_path
+            }),
+            stderr: StreamRedirection::None,
+            stdout: StreamRedirection::None,
+        });
+
+    if !command_exit_status.is_success() {
+        error!("Audio extraction failed: {:?}", command_exit_status);
+        return Err(ProcessSingleJobError::Other(anyhow!("Command failed: {:?}", command_exit_status)));
+    }
+    Ok(())
+}

crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/video_style_transfer/video_paths.rs

@@ -14,6 +14,9 @@ pub struct VideoPaths {
   /// We'll use this downstream once it's available.
   pub trimmed_resampled_video_path: PathBuf,
 
+  /// Filesystem path of the trimmed audio
+  pub trimmed_audio_path: PathBuf,
+
   /// This is the input into Comfy.
   /// This is typically the `trimmed_resampled_video_path`, but since Comfy
   /// can overwrite the source, we'll keep a separate copy of that file for
@@ -47,12 +50,14 @@ impl VideoPaths {
 
     let original_video_path = input_dir.join("video.mp4");
     let trimmed_resampled_video_path = input_dir.join("trimmed.mp4");
+    let trimmed_audio_path = input_dir.join("trimmed.wav");
     let comfy_input_video_path = input_dir.join("input.mp4");
     let comfy_output_video_path = output_dir.join(job_output_path); // TODO: This sucks.
 
     Self {
       original_video_path,
       trimmed_resampled_video_path,
+      trimmed_audio_path,
       comfy_input_video_path,
       comfy_output_video_path,
       audio_restored_video_path: None,
@@ -86,6 +91,8 @@ impl VideoPaths {
       - original video path (exists): {:?}
       - trimmed video path: {:?}
       - trimmed video path (exists): {:?}
+      - trimmed audio path: {:?}
+      - trimmed audio path (exists): {:?}
       - comfy output path: {:?}
       - comfy output path (exists): {:?}
     "#,
@@ -93,6 +100,8 @@ impl VideoPaths {
         file_exists(&self.original_video_path),
         &self.trimmed_resampled_video_path,
         file_exists(&self.trimmed_resampled_video_path),
+        &self.trimmed_audio_path,
+        file_exists(&self.trimmed_audio_path),
         &self.comfy_output_video_path,
         file_exists(&self.comfy_output_video_path),
     );
@@ -104,6 +113,8 @@ impl VideoPaths {
       - original video path (exists): {:?}
       - trimmed video path: {:?}
       - trimmed video path (exists): {:?}
+      - trimmed audio path: {:?}
+      - trimmed audio path (exists): {:?}
       - comfy input path: {:?}
       - comfy input path (exists): {:?}
       - comfy output path: {:?}
@@ -113,6 +124,8 @@ impl VideoPaths {
         file_exists(&self.original_video_path),
         &self.trimmed_resampled_video_path,
         file_exists(&self.trimmed_resampled_video_path),
+        &self.trimmed_audio_path,
+        file_exists(&self.trimmed_audio_path),
         &self.comfy_input_video_path,
         file_exists(&self.comfy_input_video_path),
         &self.comfy_output_video_path,
@@ -126,6 +139,8 @@ impl VideoPaths {
       - original video path (exists): {:?}
       - trimmed video path: {:?}
       - trimmed video path (exists): {:?}
+      - trimmed audio path: {:?}
+      - trimmed audio path (exists): {:?}
       - comfy input path: {:?}
       - comfy input path (exists): {:?}
       - comfy output path: {:?}
@@ -135,6 +150,8 @@ impl VideoPaths {
         file_exists(&self.original_video_path),
         &self.trimmed_resampled_video_path,
         file_exists(&self.trimmed_resampled_video_path),
+        &self.trimmed_audio_path,
+        file_exists(&self.trimmed_audio_path),
         &self.comfy_input_video_path,
         file_exists(&self.comfy_input_video_path),
         &self.comfy_output_video_path,
@@ -148,6 +165,8 @@ impl VideoPaths {
       - original video path (exists): {:?}
       - trimmed video path: {:?}
       - trimmed video path (exists): {:?}
+      - trimmed audio path: {:?}
+      - trimmed audio path (exists): {:?}
       - comfy output path: {:?}
       - comfy output path (exists): {:?}
       - restored audio output path: {:?}
@@ -159,6 +178,8 @@ impl VideoPaths {
         file_exists(&self.original_video_path),
         &self.trimmed_resampled_video_path,
         file_exists(&self.trimmed_resampled_video_path),
+        &self.trimmed_audio_path,
+        file_exists(&self.trimmed_audio_path),
         &self.comfy_output_video_path,
         file_exists(&self.comfy_output_video_path),
         &self.audio_restored_video_path,

crates/service/job/inference_job/src/util/common_commands/ffmpeg_command_runner.rs

@@ -1,16 +1,7 @@
-use std::collections::HashSet;
-use std::path::{Path, PathBuf};
-use std::time::Duration;
-
-use anyhow::anyhow;
-use log::{info, warn};
-use once_cell::sync::Lazy;
-use subprocess::{Popen, PopenConfig};
-
+use log::warn;
 use errors::AnyhowResult;
-use filesys::path_to_string::path_to_string;
+
 use subprocess_common::command_exit_status::CommandExitStatus;
-use subprocess_common::command_runner::command_args::CommandArgs;
 use subprocess_common::command_runner::command_runner::CommandRunner;
 use subprocess_common::command_runner::command_runner_args::RunAsSubprocessArgs;
 use subprocess_common::command_runner::env_var_policy::EnvVarPolicy;

crates/service/job/inference_job/src/util/common_commands/ffmpeg_extract_audio_args.rs

@@ -0,0 +1,28 @@
+use std::path::Path;
+
+use filesys::path_to_string::path_to_string;
+use subprocess_common::command_runner::command_args::CommandArgs;
+
+pub struct FfmpegExtractAudioArgs<'a> {
+    pub input_video_file: &'a Path,
+    pub output_file: &'a Path,
+}
+
+impl CommandArgs for FfmpegExtractAudioArgs<'_> {
+    fn to_command_string(&self) -> String {
+        let mut command = String::new();
+        command.push_str(" -y ");
+
+        // Kasisnu(2021-09-29): inspiration - https://stackoverflow.com/a/63237888/1250138
+        // ffmpeg -i input.mov -map 0:a -c copy output.mov
+        command.push_str(" -i ");
+        command.push_str(&path_to_string(self.input_video_file));
+
+        command.push_str(" -c copy ");
+        command.push_str(" -map 0:a ");
+
+        command.push_str(&path_to_string(self.output_file));
+
+        command
+    }
+}

crates/service/job/inference_job/src/util/common_commands/ffprobe_command_runner.rs

@@ -1,16 +1,8 @@
-use std::collections::HashSet;
-use std::path::{Path, PathBuf};
-use std::time::Duration;
 
-use anyhow::anyhow;
-use log::{info, warn};
-use once_cell::sync::Lazy;
-use subprocess::{Popen, PopenConfig};
+use log::warn;
 
 use errors::AnyhowResult;
-use filesys::path_to_string::path_to_string;
 use subprocess_common::command_exit_status::CommandExitStatus;
-use subprocess_common::command_runner::command_args::CommandArgs;
 use subprocess_common::command_runner::command_runner::CommandRunner;
 use subprocess_common::command_runner::command_runner_args::RunAsSubprocessArgs;
 use subprocess_common::command_runner::env_var_policy::EnvVarPolicy;

crates/service/job/inference_job/src/util/common_commands/mod.rs

@@ -1,6 +1,7 @@
 pub mod audiowmark;
 pub mod ffmpeg_audio_replace_args;
 pub mod ffmpeg_command_runner;
+pub mod ffmpeg_extract_audio_args;
 pub mod ffmpeg_logo_watermark_command;
 pub mod ffprobe_command_runner;
 pub mod ffprobe_get_dimensions_args;