initial commit of so-vits-svc cli

Cargo.toml

@@ -12,6 +12,7 @@ members = [
   "crates/cli/database_backup",
   "crates/cli/database_cleanup",
   "crates/cli/scrape_test",
+  "crates/cli/so_vits_svc_onnx_inference",
 
   # Shared libraries
   "crates/lib/actix_helpers",

crates/cli/so_vits_svc_onnx_inference/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "so_vits_svc_onnx_inference"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+clap = { version = "4.2.1", features = ["derive"] }
+env_logger = "0.10.0"
+log = "0.4.17"
+ndarray = "0.15.6"
+onnxruntime = { git = "https://github.com/28Smiles/onnxruntime-rs", rev = "aea2c8e8f16496417cbdea5dac64ce569d0060d1"}
+rand = "0.8.5"
+rsworld = "0.1.0"
+rsworld-sys = "0.1.0"
+rubato = "0.12.0"
+symphonia =  { version = "0.5.2", features = ["aac", "alac", "flac", "mp3", "ogg", "vorbis", "wav"] }
+webrtc-vad = "0.4.0"
+media = { path = "../../lib/media" }
+hound = "3.5.0"

crates/cli/so_vits_svc_onnx_inference/src/main.rs

@@ -0,0 +1,218 @@
+use std::{path::Path, collections::VecDeque};
+
+use clap::Parser;
+use log::info;
+use ndarray::{Dim, IxDynImpl, Ix3};
+use onnxruntime::{environment::Environment, LoggingLevel, GraphOptimizationLevel, tensor::OrtOwnedTensor, TypedArray};
+use rand::Fill;
+use rsworld::{dio, stonemask};
+use rsworld_sys::DioOption;
+use media::{decode::decode_audio_to_pcm_f32::decode_audio_to_pcm_f32, resample::{Resampler, copy_planar}};
+use symphonia::core::audio::{SignalSpec, Channels};
+use webrtc_vad::{Vad, SampleRate, VadMode};
+
+
+#[derive(Parser, Debug)]
+struct Args {
+    /// Path to an onnx so-vits-svc 4.0 model
+    model_path: String,
+    /// Path to an onnx hubert model
+    hubert_path: String,
+    /// Path to the audio source (supports flac, ogg, mp3, wav, alac, aac)
+    audio_input_path: String,
+    /// Path to create for the audio output file (Mono WAV PCM F32 @ 44100 Hz)
+    audio_output_path: String,
+    /// For changing the pitch of the output
+    pitch_multiplier: Option<f32>
+}
+
+fn main() {
+    let args = Args::parse();
+    so_vits_svc_pipeline(args.model_path, args.hubert_path, args.audio_input_path, args.audio_output_path, args.pitch_multiplier).unwrap();
+}
+
+fn so_vits_svc_pipeline<P>(model_path: P, hubert_path: P, audio_input_path: P, audio_output_path: P, pitch_multiplier: Option<f32>) -> Result<(), onnxruntime::error::OrtError>
+    where P: AsRef<Path>
+{
+    let environment = Environment::builder()
+    .with_log_level(LoggingLevel::Verbose)
+    .build()?;
+
+
+    let mut hubert =  environment
+        .new_session_builder()?
+        .with_optimization_level(GraphOptimizationLevel::All)?
+        .with_model_from_file(hubert_path)?;
+
+
+    let mut so_vits_svc = environment.new_session_builder()?
+        .with_optimization_level(GraphOptimizationLevel::All)?
+        .with_model_from_file(model_path)?;
+
+    let (audio, spec) = decode_audio_to_pcm_f32(audio_input_path.as_ref()); 
+    let spec = if spec.is_none() {
+        panic!("Decoding error");
+    } else {
+        spec.unwrap()
+    };
+    
+    let mut audio_441;
+    if spec.rate != 44100 {
+        let mut resampler = Resampler::new(spec, 44100, 88200);
+        audio_441 = Vec::new();
+        let mut planar = Vec::new();
+        copy_planar(&audio, &mut planar, spec.channels.count());
+        let chunk1 = resampler.resample(planar).unwrap();
+        audio_441.extend_from_slice(chunk1);
+        loop {
+            if let Some(data) = resampler.flush() {
+                audio_441.extend_from_slice(data);
+            } else {
+                break;
+            }
+        }
+    } else {
+        audio_441 = audio;
+    }
+    audio_441 = mix_n_channels(spec.channels.count(), &mut audio_441.into());
+
+    let spec = SignalSpec::new(44100, Channels::FRONT_CENTRE);
+    let mut resampler = Resampler::new(spec, 16000, 32000);
+    let mut audio_16k = Vec::new();
+    let chunk1 = resampler.resample(vec![audio_441.clone()]).unwrap();
+    audio_16k.extend_from_slice(chunk1);
+    loop {
+        if let Some(data) = resampler.flush() {
+            audio_16k.extend_from_slice(data);
+        } else {
+            break;
+        }
+    }
+
+
+    let mut vad = Vad::new_with_rate_and_mode(SampleRate::Rate16kHz, VadMode::Aggressive);
+    let vad_bools = compute_vad(&mut vad, &audio_16k);
+    let audio16k_len = audio_16k.len();
+    let tensor16k = vec![ndarray::arr1(&audio_16k).into_shape([1, 1, audio16k_len]).unwrap()];
+
+    let hubert_output: Vec<OrtOwnedTensor<f32, Dim<IxDynImpl>>> = hubert.run(tensor16k)?;
+    let hubert_shape = hubert_output[0].shape();
+    println!("hubert_shape: {:?}", hubert_shape);
+    let hubert_output = hubert_output[0].clone().into_owned().into_dimensionality::<Ix3>().unwrap();
+
+    let double_audio441: Vec<f64> = audio_441.iter().map(|float| (*float).into()).collect();
+
+
+    let (t, f0) = dio(&double_audio441, 44100, &DioOption { f0_floor: 50.0, f0_ceil:1100.0, channels_in_octave:2.0, frame_period: 1000.0*512.0/44100.0, speed: 1, allowed_range: 0.1});
+    let f0_vec = stonemask(&double_audio441, 44100, &t, &f0);
+    let f0_vec: Vec<f32> = f0_vec.iter().map(|d| *d as f32 * pitch_multiplier.unwrap_or(1.0)).collect();
+    info!("Average F0: {}", f0_vec.iter().sum::<f32>() / f0_vec.len() as f32);
+    let f0_len = f0_vec.len();
+    info!("f0_len: {}", f0.len());
+
+    let uv = ndarray::arr1(&f0_vec.clone().iter().map(|f0| if *f0 == 0.0 { 0.0 } else { 1.0 }).collect::<Vec<f32>>()).into_shape((1, f0_len)).unwrap().into_dyn();
+
+    let pitch = ndarray::Array::from_shape_vec(Dim([1, f0_len]), f0_vec).unwrap().into_dyn();
+
+    let hubert12 = hubert_output.into_shape([hubert_shape[1], hubert_shape[2]]).unwrap();
+    let hubert_expanded = repeat_expand_2d(hubert12, f0_len);
+    let hubert012 = hubert_expanded.into_shape([1, f0_len, hubert_shape[2]]).unwrap().into_dyn();
+
+    let mel2ph = (0..f0_len as i64).into_iter().collect::<ndarray::Array1<i64>>().into_shape([1, f0_len]).unwrap().into_dyn();
+
+    let mut rng = rand::thread_rng();
+    let mut random = vec![0.0f32; f0_len*192];
+    random.try_fill(&mut rng).unwrap();
+    let noise = ndarray::Array::from_shape_vec([1, 192, f0_len], random).unwrap().into_dyn();
+
+    let speaker = ndarray::arr1(&[0]).into_dyn();
+
+
+    let svc_inputs = vec![TypedArray::F32(hubert012), TypedArray::F32(pitch), TypedArray::I64(mel2ph), TypedArray::F32(uv), TypedArray::F32(noise), TypedArray::I64(speaker)];
+
+    let outputs = so_vits_svc.run_mixed::<f32, Dim<IxDynImpl>>(svc_inputs).unwrap();
+
+     let f = std::fs::File::create(audio_output_path).unwrap();
+     let spec = hound::WavSpec {
+        channels: 1,
+        sample_rate: 44100 as u32,
+        bits_per_sample: 32,
+        sample_format: hound::SampleFormat::Float,
+    };
+
+
+    let mut output_wav = outputs[0].as_slice().unwrap().to_vec();
+    replace_inactivity_with_silence_rate(&vad_bools, &mut output_wav, 2.75625);
+
+    let mut writer = hound::WavWriter::new(f, spec).unwrap();
+    for sample in output_wav.iter() {
+        writer.write_sample(*sample).unwrap();
+    }
+    writer.finalize().unwrap();
+
+
+    Ok(())
+}
+
+// some kind of janky repeat/copy/interpolate function I stole from so_vits_svc_fork
+// could probably be implemented cleaner
+fn repeat_expand_2d(content: ndarray::Array2<f32>, target_len: usize) -> ndarray::Array2<f32> {
+    let src_len =  content.shape()[0];
+    let dim0 = content.shape()[1];
+    let mut target = ndarray::Array::zeros((target_len, dim0));
+    let temp = ndarray::Array1::linspace(0.0, (src_len+1) as f32, src_len+1);
+    let temp = temp * (target_len as f64 / src_len as f64) as f32;
+    let mut current_pos = 0;
+    for i in 0..target_len {
+        if (i as f32) < temp[current_pos + 1] {
+            target.slice_mut(ndarray::s!(i, ..)).assign(&content.slice(ndarray::s!(current_pos, ..)));
+        } else {
+            current_pos += 1;
+            target.slice_mut(ndarray::s!(i, ..)).assign(&content.slice(ndarray::s!(current_pos, ..)));
+        }
+    }
+    target
+}
+
+pub fn mix_n_channels<'a>(n_channels: usize, input: &mut VecDeque<f32>) -> Vec<f32> {
+    let mut output = Vec::new();
+    for _ in  0..input.len() / n_channels {
+        let mut mixed: f32 = 0.0;
+        for _ in 0..n_channels {
+            // not sure whether this reverses the input or not, take note!
+            mixed += input.pop_front().unwrap_or(0.0);
+        }
+        output.push(mixed);
+    }
+    output
+}
+
+pub fn compute_vad<'a>(vad: &'a mut Vad, buf: &'a [f32]) -> Vec<bool> {
+    let mut vad_bools = Vec::new();
+
+    buf.chunks(VAD_SIZE).for_each(|vad_chunk| {
+        vad_bools.push(
+            vad.is_voice_segment(
+                &vad_chunk.iter().map(|a: &f32| float_to_pcm_s16(*a)).collect::<Vec<i16>>()
+            ).unwrap_or(false)
+        )
+    });
+    vad_bools
+}
+
+pub fn replace_inactivity_with_silence_rate<'a>(vad_bools: &'a Vec<bool>, buf: &'a mut [f32], rate: f32) {
+    for (i, chunk) in buf.chunks_mut((VAD_SIZE as f32 * rate) as usize).enumerate() {
+        if i >= vad_bools.len() { break };
+        for sample in chunk {
+            if !vad_bools[i] {
+                *sample = 0.0
+            }
+        }
+    }
+}
+
+pub fn float_to_pcm_s16(float: f32) -> i16 {
+    (float * 32768.0) as i16
+}
+
+const VAD_SIZE: usize = 160;

crates/lib/media/Cargo.toml

@@ -13,12 +13,15 @@ name = "media" # NB: Public "media" crate was squatted, so we're good to use thi
 path = "src/lib.rs"
 
 [dependencies]
+arrayvec = "0.7.2"
 
 # Internal
 errors = { path = "../errors" }
+hound = "3.5.0"
 
 # External
 log = "0.4.14"
+rubato = "0.12.0"
 
 # Powerful audio decoder and demuxer
 symphonia = { version = "0.5.1", features = ["aac", "alac", "flac", "isomp4", "mkv", "mp3", "ogg", "vorbis", "wav"] }

crates/lib/media/src/decode/decode_audio_to_pcm_f32.rs

@@ -0,0 +1,130 @@
+use std::{fs::File, path::Path};
+
+use symphonia::core::audio::{SampleBuffer, SignalSpec};
+use symphonia::core::codecs::DecoderOptions;
+use symphonia::core::errors::Error;
+use symphonia::core::formats::FormatOptions;
+use symphonia::core::io::MediaSourceStream;
+use symphonia::core::meta::MetadataOptions;
+use symphonia::core::probe::Hint;
+
+/// Takes a path to an audio file and returns floating point PCM data, number of channels, and sample rate
+pub fn decode_audio_to_pcm_f32<P>(audio_path: P) -> (Vec<f32>, Option<SignalSpec>) 
+    where P: AsRef<Path>
+{
+    let file = Box::new(File::open(audio_path.as_ref()).unwrap());
+
+    let mss = MediaSourceStream::new(file, Default::default());
+
+    // Create a hint to help the format registry guess what format reader is appropriate. In this
+    // example we'll leave it empty.
+    let hint = Hint::new();
+
+    // Use the default options when reading and decoding.
+    let format_opts: FormatOptions = Default::default();
+    let metadata_opts: MetadataOptions = Default::default();
+    let decoder_opts: DecoderOptions = Default::default();
+
+    // Probe the media source stream for a format.
+    let probed =
+        symphonia::default::get_probe().format(&hint, mss, &format_opts, &metadata_opts).unwrap();
+
+    // Get the format reader yielded by the probe operation.
+    let mut format = probed.format;
+
+    // Get the default track.
+    let track = format.default_track().unwrap();
+
+    // Create a decoder for the track.
+    let mut decoder =
+        symphonia::default::get_codecs().make(&track.codec_params, &decoder_opts).unwrap();
+
+    // Store the track identifier, we'll use it to filter packets.
+    let track_id = track.id;
+
+    let mut sample_buf = None;
+
+    let mut sample_vec: Vec<f32> = Vec::new();
+    let mut sample_spec: Option<SignalSpec> = None;
+
+    loop {
+        // Get the next packet from the format reader.
+        let packet = format.next_packet();
+        let packet =  match packet {
+            Ok(packet) => packet,
+            Err(_) => { break; }
+        };
+
+        // If the packet does not belong to the selected track, skip it.
+        if packet.track_id() != track_id {
+            continue;
+        }
+
+        // Decode the packet into audio samples, ignoring any decode errors.
+        match decoder.decode(&packet) {
+            Ok(audio_buf) => {
+                // The decoded audio samples may now be accessed via the audio buffer if per-channel
+                // slices of samples in their native decoded format is desired. Use-cases where
+                // the samples need to be accessed in an interleaved order or converted into
+                // another sample format, or a byte buffer is required, are covered by copying the
+                // audio buffer into a sample buffer or raw sample buffer, respectively. In the
+                // example below, we will copy the audio buffer into a sample buffer in an
+                // interleaved order while also converting to a f32 sample format.
+
+                // If this is the *first* decoded packet, create a sample buffer matching the
+                // decoded audio buffer format.
+                if sample_buf.is_none() {
+                    // Get the audio buffer specification.
+                    sample_spec = Some(*audio_buf.spec());
+
+                    // Get the capacity of the decoded buffer. Note: This is capacity, not length!
+                    let duration = audio_buf.capacity() as u64;
+
+                    // Create the f32 sample buffer.
+                    sample_buf = Some(SampleBuffer::<f32>::new(duration, sample_spec.unwrap()));
+                }
+
+                // Copy the decoded audio buffer into the sample buffer in an interleaved format.
+                if let Some(buf) = &mut sample_buf {
+                    buf.copy_interleaved_ref(audio_buf);
+                    sample_vec.extend_from_slice(buf.samples());
+                }
+            }
+            Err(Error::DecodeError(_)) => (),
+            Err(_) => break,
+        }
+    }
+    (sample_vec, sample_spec)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::path::PathBuf;
+    use super::decode_audio_to_pcm_f32;
+
+    fn test_file(path_from_repo_root: &str) -> PathBuf {
+        // https://doc.rust-lang.org/cargo/reference/environment-variables.html
+        let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+        path.push(format!("../../../{}", path_from_repo_root));
+        path
+    }
+
+    #[test]
+    fn test_flac_decode() {
+        let path = test_file("test_data/audio/flac/zelda_ocarina_small_item.flac");
+        let (buf, spec) = decode_audio_to_pcm_f32(path);
+        assert_eq!(buf.len(), 451584);
+        assert_eq!(spec.unwrap().channels.count(), 2);
+        assert_eq!(spec.unwrap().rate, 44100);
+    }
+
+    #[test]
+    fn test_ogg_decode() {
+        let path = test_file("test_data/audio/ogg/banjo-kazooie_jiggy_appearance.ogg");
+        let (buf, spec) = decode_audio_to_pcm_f32(path);
+        // disagrees with audacity but I guess ogg is just weird like that
+        assert_eq!(buf.len(), 432512);
+        assert_eq!(spec.unwrap().channels.count(), 2);
+        assert_eq!(spec.unwrap().rate, 44100);
+    }
+}

crates/lib/media/src/decode/decode_basic_audio_info.rs

@@ -202,12 +202,12 @@ fn read_duration(format: &mut Box<dyn FormatReader>) -> AnyhowResult<Option<u64>
 mod tests {
   use std::path::PathBuf;
   use errors::AnyhowResult;
-  use crate::decode_basic_audio_info::decode_basic_audio_info;
+  use super::decode_basic_audio_info;
 
   fn test_file(path_from_repo_root: &str) -> PathBuf {
     // https://doc.rust-lang.org/cargo/reference/environment-variables.html
     let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-    path.push(format!("../../../{}", path_from_repo_root));
+    path.push(format!("../../../../{}", path_from_repo_root));
     path
   }
 

crates/lib/media/src/decode/mod.rs

@@ -0,0 +1,2 @@
+pub mod decode_basic_audio_info;
+pub mod decode_audio_to_pcm_f32;

crates/lib/media/src/lib.rs

@@ -18,4 +18,5 @@
 #![allow(dead_code)]
 #![allow(non_snake_case)]
 
-pub mod decode_basic_audio_info;
+pub mod decode;
+pub mod resample;

crates/lib/media/src/resample.rs

@@ -0,0 +1,185 @@
+// Symphonia
+// Copyright (c) 2019-2022 The Project Symphonia Developers.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use symphonia::core::audio::{ SignalSpec};
+use symphonia::core::conv::{FromSample, IntoSample};
+use symphonia::core::sample::Sample;
+
+pub struct Resampler<T> {
+    resampler: rubato::FftFixedIn<f32>,
+    input: Vec<Vec<f32>>,
+    output: Vec<Vec<f32>>,
+    interleaved: Vec<T>,
+    chunk_size: usize,
+}
+
+impl<T> Resampler<T>
+where
+    T: Sample + FromSample<f32> + IntoSample<f32>,
+{
+    fn resample_inner(&mut self) -> &[T] {
+        {
+            let mut input: arrayvec::ArrayVec<&[f32], 32> = Default::default();
+
+            for channel in self.input.iter() {
+                input.push(&channel[..self.chunk_size]);
+            }
+
+            // Resample.
+            rubato::Resampler::process_into_buffer(
+                &mut self.resampler,
+                &input,
+                &mut self.output,
+                None,
+            )
+            .unwrap();
+        }
+
+        // Remove consumed samples from the input buffer.
+        for channel in self.input.iter_mut() {
+            channel.drain(0..self.chunk_size);
+        }
+
+        // Interleave the planar samples from Rubato.
+        let num_channels = self.output.len();
+        self.interleaved.resize(num_channels * self.output[0].len(), T::MID);
+
+        for (i, frame) in self.interleaved.chunks_exact_mut(num_channels).enumerate() {
+            for (ch, s) in frame.iter_mut().enumerate() {
+                *s = self.output[ch][i].into_sample();
+            }
+        }
+
+        &self.interleaved
+    }
+
+
+    /// Resamples a planar/non-interleaved input.
+    ///
+    /// Returns the resampled samples in an interleaved format.
+    pub fn resample(&mut self, input: Vec<Vec<f32>>) -> Option<&[T]> 
+    where
+    {
+        // Copy samples into input buffer.
+        self.input = input.clone();
+
+        // Check if more samples are required.
+        if self.input[0].len() < self.chunk_size {
+            return None;
+        }
+
+        Some(self.resample_inner())
+    }
+
+
+    /// Resample any remaining samples in the resample buffer.
+    pub fn flush(&mut self) -> Option<&[T]> {
+        let len = self.input[0].len();
+
+        if len == 0 {
+            return None;
+        }
+
+        let partial_len = len % self.chunk_size;
+
+        if partial_len != 0 {
+            //Fill each input channel buffer with silence to the next multiple of the resampler
+            //duration.
+            for channel in self.input.iter_mut() {
+                channel.resize(len + (self.chunk_size- partial_len), f32::MID);
+            }
+        }
+
+        Some(self.resample_inner())
+    }
+}
+
+impl<T> Resampler<T>
+where
+    T: Sample + FromSample<f32> + IntoSample<f32>,
+{
+    pub fn new(spec: SignalSpec, to_sample_rate: usize, chunk_size: usize) -> Self {
+        let num_channels = spec.channels.count();
+
+        let resampler = rubato::FftFixedIn::<f32>::new(
+            spec.rate as usize,
+            to_sample_rate,
+            chunk_size,
+            2,
+            num_channels,
+        )
+        .unwrap();
+
+        let output = rubato::Resampler::output_buffer_allocate(&resampler);
+
+        let input = vec![Vec::with_capacity(chunk_size); num_channels];
+
+        Self { resampler, input, output, chunk_size, interleaved: Default::default() }
+    }
+}
+
+pub fn copy_planar(src: &[f32], dest: &mut Vec<Vec<f32>>, n_channels: usize) {
+    let n_samples = src.len();
+    let n_frames = n_samples / n_channels;
+
+    dest.resize(n_channels, Vec::new());
+    for k in 0..n_channels {
+        dest[k].resize(n_frames, 0.0);
+    };
+
+    let mut src_pos = 0;
+    for j in 0..n_frames {
+        for i in 0..n_channels {
+            dest[i][j] = src[src_pos];
+            src_pos += 1;
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::path::PathBuf;
+
+    use super::super::decode::decode_audio_to_pcm_f32::decode_audio_to_pcm_f32;
+    use super::Resampler;
+    use super::copy_planar;
+
+    fn test_file(path_from_repo_root: &str) -> PathBuf {
+        // https://doc.rust-lang.org/cargo/reference/environment-variables.html
+        let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+        path.push(format!("../../../{}", path_from_repo_root));
+        path
+    }
+
+    #[test]
+    fn test_flac_resample_16k() {
+        let path = test_file("test_data/audio/flac/zelda_ocarina_small_item.flac");
+        let (buf, spec) = decode_audio_to_pcm_f32(path);
+        assert_eq!(buf.len(), 451584);
+        let spec = spec.unwrap();
+        let chunk_size = 96000;
+        let mut resampler: Resampler<f32> = Resampler::new(spec, 16000, chunk_size);
+        let mut planar_buf = Vec::new();
+        copy_planar(&buf, &mut planar_buf, 2); 
+        let resampled = {resampler.resample(planar_buf).unwrap()};
+        let mut output = resampled.to_vec();
+        loop {
+            let new_chunk = resampler.flush();
+            if new_chunk.is_none() {
+                break; 
+            }
+            else {
+                output.extend_from_slice(new_chunk.unwrap());
+            }
+        }
+        // should be slightly larger than this due to padding
+        assert!(output.len() > 163840);
+        // but less than a whole new chunk
+        assert!(output.len() < 163840 + (96000 / (44100 / 16000)))
+    }
+
+}