funky dev setup

let existing tts enqueue route accept gptsovits
gpt-sovits: inference works
2026-10-21 09:22:41 -04:00 · 2024-07-24 16:17:50 +05:30 · 2024-07-22 19:35:01 +05:30 · 2024-07-22 18:21:36 +05:30 · 2024-07-22 18:21:36 +05:30 · 2024-07-22 18:21:36 +05:30
55 changed files with 1888 additions and 55 deletions
@@ -26,14 +26,14 @@ MIGRATION_DIRECTORY=_sql/migrations/
 # Bucket config (non-secret)
 REGION_NAME=us-east1

-TTS_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads
-TTS_DOWNLOAD_BUCKET_ROOT=/user_uploaded_tts_models
+#TTS_DOWNLOAD_BUCKET_NAME=kasisnu-t
+#TTS_DOWNLOAD_BUCKET_ROOT=/user_uploaded_tts_models

-W2L_PRIVATE_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads
-W2L_PUBLIC_DOWNLOAD_BUCKET_NAME=dev-vocodes-public
+#W2L_PRIVATE_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads
+#W2L_PUBLIC_DOWNLOAD_BUCKET_NAME=dev-vocodes-public

-PRIVATE_BUCKET_NAME=dev-vocodes-private-uploads
-PUBLIC_BUCKET_NAME=dev-vocodes-public
+#PRIVATE_BUCKET_NAME=kasisnu-test-area-private
+#PUBLIC_BUCKET_NAME=kasisnu-test-area-public

 # Bucket roots
 AUDIO_UPLOADS_BUCKET_ROOT=/user_uploaded_w2l_audio
@@ -64,7 +64,7 @@ HIFIGAN_ROOT_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-ml/tts
 HIFIGAN_VIRTUAL_ENV_ACTIVATION_COMMAND="source python/bin/activate"
 HIFIGAN_MODEL_CHECK_SCRIPT_NAME="vocodes_model_check_hifigan.py"

-DOWNLOAD_SCRIPT="/home/tensor/code/storyteller/web-downloader/.venv/bin/python3.11 /home/tensor/code/storyteller/web-downloader/download_internet_file.py"
+#; DOWNLOAD_SCRIPT="/home/tensor/code/storyteller/web-downloader/.venv/bin/python3.11 /home/tensor/code/storyteller/web-downloader/download_internet_file.py"

 TTS_INFERENCE_SIDECAR_HOSTNAME=localhost:8000

@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\nINSERT INTO users (\n  token,\n  username,\n  display_name,\n  email_address,\n  email_gravatar_hash,\n  profile_markdown,\n  profile_rendered_html,\n  user_role_slug,\n  password_hash,\n  ip_address_creation,\n  ip_address_last_login,\n  ip_address_last_update,\n  can_access_studio\n)\nVALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? , ?)\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 13
+    },
+    "nullable": []
+  },
+  "hash": "a3066dd68c77b5381104a754b2774506990f659c9a0ed2de1fcb8e6fd953e726"
+}
@@ -350,9 +350,9 @@ dependencies = [

 [[package]]
 name = "aes"
-version = "0.8.3"
+version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2"
+checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
 dependencies = [
 "cfg-if 1.0.0",
 "cipher 0.4.4",
@@ -1014,9 +1014,9 @@ dependencies = [

 [[package]]
 name = "bumpalo"
-version = "3.14.0"
+version = "3.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
+checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"

 [[package]]
 name = "bytemuck"
@@ -1389,6 +1389,12 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"

+[[package]]
+name = "constant_time_eq"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2"
+
 [[package]]
 name = "container_common"
 version = "0.0.1"
@@ -1482,9 +1488,9 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"

 [[package]]
 name = "crc32fast"
-version = "1.4.0"
+version = "1.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa"
+checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
 dependencies = [
 "cfg-if 1.0.0",
 ]
@@ -1794,6 +1800,12 @@ version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f578e8e2c440e7297e008bb5486a3a8a194775224bbc23729b0dbdfaeebf162e"

+[[package]]
+name = "deflate64"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b"
+
 [[package]]
 name = "der"
 version = "0.7.8"
@@ -3540,6 +3552,7 @@ dependencies = [
 "tts_common",
 "videos",
 "walkdir",
+ "zip 2.1.3",
 ]

 [[package]]
@@ -3899,6 +3912,12 @@ dependencies = [
 "scopeguard",
 ]

+[[package]]
+name = "lockfree-object-pool"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e"
+
 [[package]]
 name = "log"
 version = "0.4.21"
@@ -3940,6 +3959,16 @@ version = "0.11.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9106e1d747ffd48e6be5bb2d97fa706ed25b144fbee4d5c02eae110cd8d6badd"

+[[package]]
+name = "lzma-rs"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e"
+dependencies = [
+ "byteorder",
+ "crc",
+]
+
 [[package]]
 name = "mac"
 version = "0.1.1"
@@ -4912,6 +4941,16 @@ dependencies = [
 "sha2 0.10.8",
 ]

+[[package]]
+name = "pbkdf2"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2"
+dependencies = [
+ "digest 0.10.7",
+ "hmac 0.12.1",
+]
+
 [[package]]
 name = "pem-rfc7468"
 version = "0.7.0"
@@ -8682,9 +8721,23 @@ dependencies = [

 [[package]]
 name = "zeroize"
-version = "1.7.0"
+version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d"
+checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
+dependencies = [
+ "zeroize_derive",
+]
+
+[[package]]
+name = "zeroize_derive"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.66",
+]

 [[package]]
 name = "zerovec"
@@ -8714,15 +8767,15 @@ version = "0.6.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261"
 dependencies = [
- "aes 0.8.3",
+ "aes 0.8.4",
 "byteorder",
 "bzip2",
- "constant_time_eq",
+ "constant_time_eq 0.1.5",
 "crc32fast",
 "crossbeam-utils 0.8.20",
 "flate2",
 "hmac 0.12.1",
- "pbkdf2",
+ "pbkdf2 0.11.0",
 "sha1 0.10.6",
 "time",
 "zstd 0.11.2+zstd.1.5.2",
@@ -8744,6 +8797,49 @@ dependencies = [
 "thiserror",
 ]

+[[package]]
+name = "zip"
+version = "2.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "775a2b471036342aa69bc5a602bc889cb0a06cda00477d0c69566757d5553d39"
+dependencies = [
+ "aes 0.8.4",
+ "arbitrary 1.3.2",
+ "bzip2",
+ "constant_time_eq 0.3.0",
+ "crc32fast",
+ "crossbeam-utils 0.8.20",
+ "deflate64",
+ "displaydoc",
+ "flate2",
+ "hmac 0.12.1",
+ "indexmap 2.2.6",
+ "lzma-rs",
+ "memchr",
+ "pbkdf2 0.12.2",
+ "rand 0.8.5",
+ "sha1 0.10.6",
+ "thiserror",
+ "time",
+ "zeroize",
+ "zopfli",
+ "zstd 0.13.1",
+]
+
+[[package]]
+name = "zopfli"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946"
+dependencies = [
+ "bumpalo",
+ "crc32fast",
+ "lockfree-object-pool",
+ "log",
+ "once_cell",
+ "simd-adler32",
+]
+
 [[package]]
 name = "zstd"
 version = "0.11.2+zstd.1.5.2"
@@ -0,0 +1,47 @@
+# -*- mode: Python -*-
+
+if k8s_context() != 'black1':
+  fail("failing early to avoid overwriting prod")
+local_resource('storyteller-web-binary',
+cmd='SQLX_OFFLINE=true cargo build --release --bin storyteller-web --target=x86_64-unknown-linux-musl',
+)
+
+allow_k8s_contexts('black1')
+sync_src_crates = sync('./crates', '/storyteller-rust/source/crates')
+sync_src_sql = sync('./_sql', '/storyteller-rust/source/_sql')
+sync_src_cargo_lock = sync('./Cargo.lock', '/storyteller-rust/source/Cargo.lock')
+sync_src_cargo = sync('./Cargo.toml', '/storyteller-rust/source/Cargo.toml')
+sync_src_env = sync('./.env', '/storyteller-rust/source/.env')
+sync_src_startup = sync('./_develop/localdev/startup.sh', '/storyteller-rust/_develop/localdev/startup.sh')
+sync_src_storyteller_binary = sync('./target/x86_64-unknown-linux-musl/release/storyteller-web', '/storyteller-rust/target/x86_64-unknown-linux-musl/release/storyteller-web')
+docker_build('storyteller-web',
+ '.',
+ dockerfile='_develop/localdev/Dockerfile',
+ platform='linux/amd64',
+ only=[
+ './target/x86_64-unknown-linux-musl/release/storyteller-web',
+ '_develop/localdev/Dockerfile',
+ './_develop/localdev/startup.sh',
+ 'crates/service/web/storyteller_web/config/storyteller-web.development.env',
+ '_sql', 'diesel.toml', '.env'],
+ live_update = [
+    sync_src_crates,
+    sync_src_sql,
+    sync_src_cargo_lock,
+    sync_src_cargo,
+    sync_src_env,
+    sync_src_startup,
+    sync_src_storyteller_binary,
+    run('date > /restart.txt')
+     ],
+ )
+k8s_yaml('_develop/localdev/kubernetes.yml',
+
+)
+k8s_resource(
+  objects=['ssd-hostpath:storageclass', 'test-pvc:persistentvolumeclaim'],
+  new_name='storyteller-web-pvc',
+  trigger_mode=TRIGGER_MODE_MANUAL,
+)
+k8s_resource('mysql', port_forwards='3307:3306')
+k8s_resource('storyteller-web', port_forwards=12345)
@@ -6,9 +6,20 @@ RUN apt-get update && \
    apt-get install -y \
        mariadb-client \
        imagemagick  \
+        python3-pip \
        libmariadb-dev \
    && rm -rf /var/lib/apt/lists/*

+RUN cd /tmp && \
+    wget -O entr.zip https://github.com/eradman/entr/archive/refs/tags/5.6.zip && \
+    unzip entr.zip && \
+    cd entr-5.6 && \
+    ./configure && \
+    make test && \
+    make install \
+    && rm -rf /tmp/entr-5.6
+
+

 RUN cargo install diesel_cli --features mysql && \
  cargo install sqlx-cli --features mysql
@@ -19,5 +30,5 @@ ARG GIT_SHA

 RUN echo -n ${GIT_SHA} > /GIT_SHA

-CMD ["/storyteller-rust/localdev/startup.sh"]
+CMD ["/storyteller-rust/_develop/localdev/startup.sh"]

@@ -27,6 +27,38 @@ spec:
        ports:
          - containerPort: 3306 

+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ssd-hostpath
+provisioner: microk8s.io/hostpath
+reclaimPolicy: Retain
+volumeBindingMode: WaitForFirstConsumer
+parameters:
+  pvDir: /home/kasisnu/microk8s/hostpath-provisioner
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: test-pvc
+spec:
+  storageClassName: ssd-hostpath
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: storyteller-config
+data:
+  buckets.vocodes_bucket_region: us-east1
+  buckets.vocodes_private_bucket_name: kasisnu-test-area-private
+  buckets.vocodes_public_bucket_name: kasisnu-test-area-public
+
 ---

 apiVersion: apps/v1
@@ -65,29 +97,57 @@ spec:
      labels:
        app: storyteller-web
    spec:
+      runtimeClassName: nvidia
      containers:
+      - name: gpt-sovitsdir
+        image: localhost:32000/some-gpt-sovits:latest
+        imagePullPolicy: Always
+        command: ["sleep", "infinity"]
+        resources:
+          limits:
+            nvidia.com/gpu: "1"
+          requests:
+            nvidia.com/gpu: "1"
+        volumeMounts:
+          - mountPath: /application-dir
+            name: application-dir
      - name: storyteller-web
        image: storyteller-web:latest
-        imagePullPolicy: Never
+        imagePullPolicy: Always
        env:
          - name: MYSQL_URL
-            value: mysql://storyteller:password@mysql.default/storyteller
+            value: mysql://storyteller:password@mysql/storyteller
          - name: REDIS_0_URL
-            value: redis://redisd@redis.default/0
+            value: redis://redisd@redis/0
          - name: REDIS_1_URL
-            value: redis://redisd@redis.default/1
+            value: redis://redisd@redis/1
          - name: ACCESS_KEY
-            value: garbage
+            valueFrom:
+              secretKeyRef:
+                name: bucket-secrets
+                key: access-key
          - name: SECRET_KEY
-            value: garbage
+            valueFrom:
+              secretKeyRef:
+                name: bucket-secrets
+                key: secret-key
          - name: REGION_NAME
-            value: garbage
+            valueFrom:
+              configMapKeyRef:
+                name: storyteller-config
+                key: buckets.vocodes_bucket_region
          - name: W2L_PRIVATE_DOWNLOAD_BUCKET_NAME
-            value: garbage
+            valueFrom:
+              configMapKeyRef:
+                name: storyteller-config
+                key: buckets.vocodes_private_bucket_name
          - name: W2L_PUBLIC_DOWNLOAD_BUCKET_NAME
-            value: garbage
+            valueFrom:
+              configMapKeyRef:
+                name: storyteller-config
+                key: buckets.vocodes_public_bucket_name
          - name: AUDIO_UPLOADS_BUCKET_ROOT
-            value: garbage
+            value: /user_uploaded_w2l_audio
          - name: TWITCH_USERNAME
            value: garbage
          - name: TWITCH_STREAM_KEY
@@ -101,10 +161,14 @@ spec:
        volumeMounts:
          - mountPath: /storyteller-rust
            name: storyteller-rust-folder
+          - mountPath: /application-dir
+            name: application-dir
      volumes:
        - name: storyteller-rust-folder
-          hostPath:
-            path: /storyteller-rust
+          persistentVolumeClaim:
+            claimName: test-pvc
+        - name: application-dir
+          emptyDir: {}
 ---

 apiVersion: v1
@@ -116,7 +180,7 @@ spec:
  ports:
    - name: mysql
      port: 3306
-      nodePort: 3306
+      nodePort: 32206
  selector:
    app: mysql

@@ -131,7 +195,7 @@ spec:
  ports:
    - name: redis
      port: 6379
-      nodePort: 6379
+      nodePort: 32379
  selector:
    app: redis

@@ -146,7 +210,7 @@ spec:
  ports:
    - name: storyteller-web
      port: 12345
-      nodePort: 12345
+      nodePort: 32345
  selector:
    app: storyteller-web

@@ -2,5 +2,8 @@
 set +e +o pipefail
 diesel migration run --database-url="${MYSQL_URL}"
 set -e -o pipefail
-./target/x86_64-unknown-linux-musl/release/storyteller-web
-
+echo 'hello world in startup.sh'
+#sleep infinity
+date > /restart.txt
+echo /restart.txt | entr -nrz /storyteller-rust/target/x86_64-unknown-linux-musl/release/storyteller-web
+#/storyteller-rust/target/x86_64-unknown-linux-musl/release/storyteller-web
@@ -24,6 +24,9 @@ pub enum InferenceJobType {
  /// These run in Comfy.
  LivePortrait,

+  /// Voice jobs that use GPT-Sovits
+  GptSovits,
+
  /// Jobs that run ComfyUI workflows
  /// This is actually just for Video Style Transfer and Storyteller Studio.
  #[deprecated(note = "Use VideoRender instead.")]
@@ -85,6 +88,7 @@ impl InferenceJobType {
    match self {
      Self::VideoRender => "video_render",
      Self::LivePortrait => "live_portrait",
+      Self::GptSovits => "gpt_sovits",
      Self::ComfyUi => "comfy_ui",
      Self::ConvertFbxToGltf => "convert_fbx_gltf",
      Self::MocapNet => "mocap_net",
@@ -104,6 +108,7 @@ impl InferenceJobType {
    match value {
      "video_render" => Ok(Self::VideoRender),
      "live_portrait" => Ok(Self::LivePortrait),
+      "gpt_sovits" => Ok(Self::GptSovits),
      "comfy_ui" => Ok(Self::ComfyUi),
      "convert_fbx_gltf" => Ok(Self::ConvertFbxToGltf),
      "mocap_net" => Ok(Self::MocapNet),
@@ -126,6 +131,7 @@ impl InferenceJobType {
    BTreeSet::from([
      Self::VideoRender,
      Self::LivePortrait,
+      Self::GptSovits,
      Self::ComfyUi,
      Self::ConvertFbxToGltf,
      Self::MocapNet,
@@ -159,6 +165,7 @@ mod tests {
    fn test_serialization() {
      assert_serialization(InferenceJobType::VideoRender, "video_render");
      assert_serialization(InferenceJobType::LivePortrait, "live_portrait");
+      assert_serialization(InferenceJobType::GptSovits, "gpt_sovits");
      assert_serialization(InferenceJobType::ComfyUi, "comfy_ui");
      assert_serialization(InferenceJobType::ConvertFbxToGltf, "convert_fbx_gltf");
      assert_serialization(InferenceJobType::MocapNet, "mocap_net");
@@ -177,6 +184,7 @@ mod tests {
    fn to_str() {
      assert_eq!(InferenceJobType::VideoRender.to_str(), "video_render");
      assert_eq!(InferenceJobType::LivePortrait.to_str(), "live_portrait");
+      assert_eq!(InferenceJobType::GptSovits.to_str(), "gpt_sovits");
      assert_eq!(InferenceJobType::ComfyUi.to_str(), "comfy_ui");
      assert_eq!(InferenceJobType::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
      assert_eq!(InferenceJobType::MocapNet.to_str(), "mocap_net");
@@ -195,6 +203,7 @@ mod tests {
    fn from_str() {
      assert_eq!(InferenceJobType::from_str("video_render").unwrap(), InferenceJobType::VideoRender);
      assert_eq!(InferenceJobType::from_str("live_portrait").unwrap(), InferenceJobType::LivePortrait);
+      assert_eq!(InferenceJobType::from_str("gpt_sovits").unwrap(), InferenceJobType::GptSovits);
      assert_eq!(InferenceJobType::from_str("comfy_ui").unwrap(), InferenceJobType::ComfyUi);
      assert_eq!(InferenceJobType::from_str("convert_fbx_gltf").unwrap(), InferenceJobType::ConvertFbxToGltf);
      assert_eq!(InferenceJobType::from_str("mocap_net").unwrap(), InferenceJobType::MocapNet);
@@ -213,9 +222,10 @@ mod tests {
    fn all_variants() {
      // Static check
      let mut variants = InferenceJobType::all_variants();
-      assert_eq!(variants.len(), 14);
+      assert_eq!(variants.len(), 15);
      assert_eq!(variants.pop_first(), Some(InferenceJobType::VideoRender));
      assert_eq!(variants.pop_first(), Some(InferenceJobType::LivePortrait));
+      assert_eq!(variants.pop_first(), Some(InferenceJobType::GptSovits));
      assert_eq!(variants.pop_first(), Some(InferenceJobType::ComfyUi));
      assert_eq!(variants.pop_first(), Some(InferenceJobType::ConvertFbxToGltf));
      assert_eq!(variants.pop_first(), Some(InferenceJobType::MocapNet));
@@ -47,6 +47,9 @@ pub enum MediaFileOriginModelType {

  #[serde(rename = "stable_diffusion_1_5")]
  StableDiffusion15,
+
+  #[serde(rename = "gpt_sovits")]
+  GptSovits,
 }

 // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests!
@@ -67,7 +70,8 @@ impl MediaFileOriginModelType {
      Self::MocapNet => "mocap_net",
      Self::ComfyUi => "comfy_ui",
      Self::StyleTTS2 => "styletts2",
-      Self::StableDiffusion15 => "stable_diffusion_1_5"
+      Self::StableDiffusion15 => "stable_diffusion_1_5",
+      Self::GptSovits => "gpt_sovits",
    }
  }

@@ -83,6 +87,7 @@ impl MediaFileOriginModelType {
      "comfy_ui" => Ok(Self::ComfyUi),
      "styletts2" => Ok(Self::StyleTTS2),
      "stable_diffusion_1_5" => Ok(Self::StableDiffusion15),
+      "gpt_sovits" => Ok(Self::GptSovits),
      _ => Err(format!("invalid value: {:?}", value)),
    }
  }
@@ -100,7 +105,8 @@ impl MediaFileOriginModelType {
      Self::MocapNet,
      Self::ComfyUi,
      Self::StyleTTS2,
-      Self::StableDiffusion15
+      Self::StableDiffusion15,
+      Self::GptSovits,
    ])
  }
 }
@@ -130,6 +136,7 @@ mod tests {
      assert_eq!(MediaFileOriginModelType::ComfyUi.to_str(), "comfy_ui");
      assert_eq!(MediaFileOriginModelType::StyleTTS2.to_str(), "styletts2");
      assert_eq!(MediaFileOriginModelType::StableDiffusion15.to_str(), "stable_diffusion_1_5");
+      assert_eq!(MediaFileOriginModelType::GptSovits.to_str(), "gpt_sovits");
  }

    #[test]
@@ -144,13 +151,14 @@ mod tests {
      assert_eq!(MediaFileOriginModelType::from_str("comfy_ui").unwrap(), MediaFileOriginModelType::ComfyUi);
      assert_eq!(MediaFileOriginModelType::from_str("styletts2").unwrap(), MediaFileOriginModelType::StyleTTS2);
      assert_eq!(MediaFileOriginModelType::from_str("stable_diffusion_1_5").unwrap(), MediaFileOriginModelType::StableDiffusion15);
+      assert_eq!(MediaFileOriginModelType::from_str("gpt_sovits").unwrap(), MediaFileOriginModelType::GptSovits);
      assert!(MediaFileOriginModelType::from_str("foo").is_err());
    }

    #[test]
    fn all_variants() {
      let mut variants = MediaFileOriginModelType::all_variants();
-      assert_eq!(variants.len(), 10);
+      assert_eq!(variants.len(), 11);
      assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::RvcV2));
      assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::SadTalker));
      assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::SoVitsSvc));
@@ -161,6 +169,7 @@ mod tests {
      assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::ComfyUi));
      assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::StyleTTS2));
      assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::StableDiffusion15));
+      assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::GptSovits));
      assert_eq!(variants.pop_first(), None);
    }
  }
@@ -23,6 +23,8 @@ pub enum PublicInferenceJobType {
  /// TODO(bt,2024-07-15): We may segregate these two job types in the future
  VideoRender,

+  GptSovits,
+
  // Everything else is the same

  /// Jobs that run ComfyUI workflows
@@ -85,6 +87,7 @@ impl PublicInferenceJobType {
      InferenceJobType::LivePortrait => Self::ActingFace,
      // Conserved variants
      InferenceJobType::VideoRender => Self::VideoRender,
+      InferenceJobType::GptSovits => Self::GptSovits,
      InferenceJobType::ComfyUi => Self::ComfyUi,
      InferenceJobType::ConvertFbxToGltf => Self::ConvertFbxToGltf,
      InferenceJobType::MocapNet => Self::MocapNet,
@@ -106,6 +109,7 @@ impl PublicInferenceJobType {
      Self::ActingFace => InferenceJobType::LivePortrait,
      // Conserved variants
      Self::VideoRender => InferenceJobType::VideoRender,
+      Self::GptSovits => InferenceJobType::GptSovits,
      Self::ComfyUi => InferenceJobType::ComfyUi,
      Self::ConvertFbxToGltf => InferenceJobType::ConvertFbxToGltf,
      Self::MocapNet => InferenceJobType::MocapNet,
@@ -16,6 +16,7 @@ pub fn get_bucket_clients() -> AnyhowResult<BucketClients> {
  let region_name = easyenv::get_env_string_required("REGION_NAME")?;
  let public_bucket_name = easyenv::get_env_string_required("PUBLIC_BUCKET_NAME")?;
  let private_bucket_name = easyenv::get_env_string_required("PRIVATE_BUCKET_NAME")?;
+  let s3_compatible_endpoint_url = easyenv::get_env_string_or_default("S3_COMPATIBLE_ENDPOINT_URL", "https://storage.googleapis.com");

  // NB: Long timeout for dev rust builds to upload to cloud buckets.
  // Unoptimized binaries sometimes take a lot of time to upload, presumably due to unoptimized code.
@@ -29,6 +30,7 @@ pub fn get_bucket_clients() -> AnyhowResult<BucketClients> {
    &secret_key,
    &region_name,
    &public_bucket_name,
+    &s3_compatible_endpoint_url,
    None,
    Some(bucket_timeout),
  )?;
@@ -40,6 +42,7 @@ pub fn get_bucket_clients() -> AnyhowResult<BucketClients> {
    &secret_key,
    &region_name,
    &private_bucket_name,
+    &s3_compatible_endpoint_url,
    None,
    Some(bucket_timeout),
  )?;
@@ -43,6 +43,8 @@ pub async fn main() -> AnyhowResult<()> {
  let region_name = easyenv::get_env_string_required("REGION_NAME")?;
  let public_bucket_name = easyenv::get_env_string_required("PUBLIC_BUCKET_NAME")?;

+  let s3_compatible_endpoint_url = easyenv::get_env_string_or_default(
+    "S3_COMPATIBLE_ENDPOINT_URL", "");
  let bucket_timeout = easyenv::get_env_duration_seconds_or_default(
    "BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 5));

@@ -53,6 +55,7 @@ pub async fn main() -> AnyhowResult<()> {
    &secret_key,
    &region_name,
    &public_bucket_name,
+    &s3_compatible_endpoint_url,
    None,
    Some(bucket_timeout),
  )?;
@@ -75,6 +75,8 @@ pub fn get_bucket_client(env_var_suffix: &str) -> AnyhowResult<BucketClient> {
  let region_name = easyenv::get_env_string_required(&format!("REGION_NAME_{env_var_suffix}"))?;
  let bucket_name = easyenv::get_env_string_required(&format!("BUCKET_NAME_{env_var_suffix}"))?;

+  let s3_compatible_endpoint_url = easyenv::get_env_string_or_default(
+    &format!("S3_COMPATIBLE_ENDPOINT_URL_{env_var_suffix}"), "https://storage.googleapis.com");
  // NB: Long timeout for dev rust builds to upload to cloud buckets.
  // Unoptimized binaries sometimes take a lot of time to upload, presumably due to unoptimized code.
  let bucket_timeout = easyenv::get_env_duration_seconds_or_default(
@@ -87,6 +89,7 @@ pub fn get_bucket_client(env_var_suffix: &str) -> AnyhowResult<BucketClient> {
    &secret_key,
    &region_name,
    &bucket_name,
+    &s3_compatible_endpoint_url,
    None,
    Some(bucket_timeout),
  )?)
@@ -47,6 +47,7 @@ impl BucketClient {
    secret_key: &str,
    region_name: &str,
    bucket_name: &str,
+    s3_endpoint: &str,
    optional_bucket_root: Option<&str>,
    // See underlying docs for timeout details.
    bucket_request_timeout: Option<Duration>,
@@ -63,14 +64,14 @@ impl BucketClient {
    // NB: The GCS buckets aren't supported by default.
    let region = Region::Custom {
      region: region_name.to_owned(),
-      endpoint: "https://storage.googleapis.com".to_owned(),
+      endpoint: s3_endpoint.to_owned(),
    };

    let mut bucket = Bucket::new(&bucket_name, region, credentials)?;

    bucket.set_request_timeout(bucket_request_timeout);
    bucket.set_path_style();
-    bucket.set_subdomain_style();
+    // bucket.set_subdomain_style();

    let optional_bucket_root = optional_bucket_root.map(|s| s.to_string());

@@ -28,6 +28,7 @@ pub struct BucketOrchestration {
    region_name: String,
    public_bucket_name: String,
    private_bucket_name: String,
+    s3_compatible_endpoint_url: String,
 }

 #[async_trait]
@@ -63,12 +64,17 @@ impl BucketOrchestration {
        let region_name = easyenv::get_env_string_required("REGION_NAME")?;
        let public_bucket_name = easyenv::get_env_string_required("PUBLIC_BUCKET_NAME")?;
        let private_bucket_name = easyenv::get_env_string_required("PRIVATE_BUCKET_NAME")?;
+        let s3_compatible_endpoint_url = easyenv::get_env_string_or_default(
+            "S3_COMPATIBLE_ENDPOINT_URL",
+            "https://storage.googleapis.com",
+        );


        let bucket_orchestration = BucketOrchestration::new(
            access_key,
            secret_key,
            region_name,
+            s3_compatible_endpoint_url,
            public_bucket_name,
            private_bucket_name,
        );
@@ -80,6 +86,7 @@ impl BucketOrchestration {
        access_key: String,
        secret_key: String,
        region_name: String,
+        s3_compatible_endpoint_url: String,
        public_bucket_name: String,
        private_bucket_name: String) -> Self {
        Self {
@@ -88,6 +95,7 @@ impl BucketOrchestration {
            region_name,
            public_bucket_name,
            private_bucket_name,
+            s3_compatible_endpoint_url,
        }
    }

@@ -102,6 +110,7 @@ impl BucketOrchestration {
                &self.access_key,
                &self.secret_key,
                &self.region_name,
+                &self.s3_compatible_endpoint_url,
                &self.public_bucket_name,
                None,
                Some(bucket_timeout),
@@ -112,6 +121,7 @@ impl BucketOrchestration {
                &self.access_key,
                &self.secret_key,
                &self.region_name,
+                &self.s3_compatible_endpoint_url,
                &self.private_bucket_name,
                None,
                Some(bucket_timeout),
@@ -10,6 +10,7 @@ pub struct SemiPersistentCacheDir {
  tts_synthesizer_model_root: PathBuf,
  tts_pretrained_vocoder_model_root: PathBuf, // Pretrained, non-user uploaded vocoders
  vocoder_model_root: PathBuf, // User-uploaded vocoders
+  gpt_sovits_model_root: PathBuf, // GPT-SoViT models

  voice_conversion_model_root: PathBuf,

@@ -37,6 +38,7 @@ impl SemiPersistentCacheDir {
      tts_synthesizer_model_root: cache_root.join("tts/synthesizer_models/"),
      tts_pretrained_vocoder_model_root: cache_root.join("tts/vocoder_models_pretrained/"),
      vocoder_model_root: cache_root.join("tts/user_uploaded_vocoder_models/"),
+      gpt_sovits_model_root: cache_root.join("tts/gpt_sovits_models/"),

      voice_conversion_model_root: cache_root.join("voice_conversion/models/"),

@@ -191,6 +193,20 @@ impl SemiPersistentCacheDir {
    Ok(())
  }

+  // ==================== GPT SOVITS MODELS ====================
+  pub fn gpt_sovits_model_path(&self, model_filename: &str) -> PathBuf {
+    self.gpt_sovits_model_root.join(model_filename)
+  }
+
+  pub fn gpt_sovits_model_directory(&self) -> &Path {
+    &self.gpt_sovits_model_root
+  }
+
+  pub fn create_gpt_sovits_model_path(&self) -> AnyhowResult<()> {
+    let _ = fs::create_dir_all(self.gpt_sovits_model_directory())?;
+    Ok(())
+  }
+
  // ==================== W2L OUTPUT RESULTS ====================

  // We cache W2L faces here.
@@ -2,6 +2,8 @@ use serde_derive::{Deserialize, Serialize};
 use sqlx::MySql;
 use sqlx::pool::PoolConnection;

+use enums::by_table::generic_inference_jobs::inference_job_type::InferenceJobType;
+use enums::by_table::model_weights::weights_types::WeightsType;
 use enums::common::visibility::Visibility;
 use errors::AnyhowResult;
 use mysql_queries::queries::model_weights::get::get_weight_for_legacy_tts_enqueue::{get_weight_for_legacy_tts_enqueue_with_connection, ModelWeightForLegacyTtsEnqueue};
@@ -72,4 +74,15 @@ impl TtsModelForEnqueueInferenceMigrationWrapper {
      Self::ModelWeight(ref model) => model.creator_set_visibility,
    }
  }
+
+  pub fn job_type(&self) -> InferenceJobType {
+    match self {
+      Self::LegacyTts(_) => InferenceJobType::Tacotron2,
+      Self::ModelWeight(ref model) => match model.weights_type {
+        WeightsType::Tacotron2 => InferenceJobType::Tacotron2,
+        WeightsType::GptSoVits => InferenceJobType::GptSovits,
+        _ => InferenceJobType::Tacotron2,
+      }
+    }
+  }
 }
@@ -1,5 +1,5 @@
 use errors::AnyhowResult;
-
+use crate::payloads::generic_inference_args::gptsovits_payload::GptSovitsPayload;
 use crate::payloads::generic_inference_args::image_generation_payload::StableDiffusionArgs;
 use crate::payloads::generic_inference_args::lipsync_payload::LipsyncArgs;
 use crate::payloads::generic_inference_args::live_portrait_payload::LivePortraitPayload;
@@ -67,6 +67,10 @@ pub enum InferenceCategoryAbbreviated {
  #[serde(rename = "bw")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
  #[serde(alias = "convert_bvh_to_workflow")]
  ConvertBvhToWorkflow,
+
+  #[serde(rename = "gs")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  #[serde(alias = "gpt_sovits")]
+  GptSovits
 }

 #[derive(Serialize, Deserialize, Clone, Copy, Debug)]
@@ -134,6 +138,9 @@ pub enum PolymorphicInferenceArgs {

  /// Render engine scene to video args
  Es(RenderEngineSceneToVideoArgs),
+
+  /// GPT Sovits
+  Gs(GptSovitsPayload),
 }

 impl GenericInferenceArgs {
@@ -0,0 +1,17 @@
+use enums::common::visibility::Visibility;
+
+#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
+pub struct GptSovitsPayload {
+  #[serde(rename = "ti")]
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub maybe_title: Option<String>,
+
+  #[serde(rename = "de")]
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub maybe_description: Option<String>,
+
+  #[serde(rename = "cv")]
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub creator_visibility: Option<Visibility>,
+}
+
@@ -7,3 +7,4 @@ pub mod render_engine_scene_to_video_payload;
 pub mod tts_payload;
 pub mod videofilter_payload;
 pub mod workflow_payload;
+pub mod gptsovits_payload;
@@ -41,6 +41,7 @@ pub struct AvailableInferenceJob {
  pub maybe_inference_args: Option<GenericInferenceArgs>,
  pub maybe_raw_inference_text: Option<String>,

+  pub maybe_download_url: Option<String>,
  // For model uploads, a possible cover image
  pub maybe_cover_image_media_file_token: Option<MediaFileToken>,

@@ -120,6 +121,7 @@ pub async fn list_available_generic_inference_jobs(
          creator_ip_address: record.creator_ip_address,
          maybe_creator_user_token: record.maybe_creator_user_token.clone(),
          maybe_creator_user_token_typed,
+          maybe_download_url: record.maybe_download_url,
          maybe_creator_anonymous_visitor_token: record.maybe_creator_anonymous_visitor_token,
          maybe_cover_image_media_file_token: record.maybe_cover_image_media_file_token
              .map(|s| MediaFileToken::new_from_str(&s)),
@@ -226,6 +228,7 @@ SELECT

  maybe_inference_args,
  maybe_raw_inference_text,
+  maybe_download_url,

  maybe_creator_user_token,
  maybe_creator_anonymous_visitor_token,
@@ -330,6 +333,7 @@ struct AvailableInferenceJobRawInternal {
  pub maybe_inference_args: Option<String>,
  pub maybe_raw_inference_text: Option<String>,

+  pub maybe_download_url: Option<String>,
  pub maybe_cover_image_media_file_token: Option<String>,

  // User information to propagate downstream
@@ -89,6 +89,8 @@ async fn main() -> AnyhowResult<()> {
  let private_bucket_name = easyenv::get_env_string_required("PRIVATE_BUCKET_NAME")?;
  let public_bucket_name = easyenv::get_env_string_required("PUBLIC_BUCKET_NAME")?;
  let bucket_root = easyenv::get_env_string_required(ENV_BUCKET_ROOT)?;
+  let s3_compatible_endpoint_url = easyenv::get_env_string_or_default("S3_COMPATIBLE_ENDPOINT_URL",
+    "https://storage.googleapis.com");

  let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS",
    Duration::from_secs(60 * 5));
@@ -98,6 +100,7 @@ async fn main() -> AnyhowResult<()> {
    &secret_key,
    &region_name,
    &private_bucket_name,
+    &s3_compatible_endpoint_url,
    None,
    Some(bucket_timeout),
  )?;
@@ -107,6 +110,7 @@ async fn main() -> AnyhowResult<()> {
    &secret_key,
    &region_name,
    &public_bucket_name,
+    &s3_compatible_endpoint_url,
    None,
    Some(bucket_timeout),
  )?;
@@ -92,3 +92,4 @@ walkdir = "2.4.0"
 opentelemetry = {version = "0.23.0", features = ["metrics"]}
 opentelemetry-otlp = { version = "0.16.0", features = ["metrics"] }
 opentelemetry_sdk = { version = "0.23.0", features = ["metrics", "rt-tokio"] }
+zip = "2.1.3"
@@ -7,7 +7,7 @@ BUCKET_TIMEOUT_SECONDS=6000
 HTTP_BIND_ADDRESS=0.0.0.0:11223

 # NB: Disable if not testing Redis
-REDIS_FOR_KEEPALIVE_URL="redis://localhost/0"
+REDIS_FOR_KEEPALIVE_URL="redis://redis/0"

 # General model weights cache directory
 MODEL_WEIGHTS_CACHE_DIR="/tmp/model_weights_cache"
@@ -193,4 +193,22 @@ HF_DATASETS_CACHE="/tmp/huggingface"
 # nltk also caches data.
 NLTK_DATA="/tmp/nltk_data"

-SKIP_TELEMETRY="true"
+SKIP_TELEMETRY="true"
+SCOPED_EXECUTION_JOB_TYPES="gpt_sovits"
+
+DOWNLOAD_SCRIPT="/storyteller-rust/downloader/download_internet_file.py"
+
+PRIVATE_BUCKET_NAME=kasisnu-test-area-private
+PUBLIC_BUCKET_NAME=kasisnu-test-area-public
+
+
+ACCESS_KEY=bWDc7MntZ0sl7TcM
+SECRET_KEY=ixbQ1QFQ8tCh5A5V2Otp6V4ANuLSn8ww
+S3_COMPATIBLE_ENDPOINT_URL=http://minio.minio-operator
+REGION_NAME=us-east1
+MYSQL_URL=mysql://storyteller:password@mysql/storyteller
+REDIS_0_URL=redis://redis@redis/0
+REDIS_1_URL=redis://redis@redis/1
+GPT_SOVITS_CODE_DIRECTORY=/model_code/GPT-SoVITS
+GPT_SOVITS_VENV_ACTIVATION_COMMAND="source /python_install/python/bin/activate"
+GPT_SOVITS_INFERENCE_COMMAND="python3 fakeyou_infer.py"
@@ -17,6 +17,7 @@ use crate::job::job_loop::job_success_result::JobSuccessResult;
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
 use crate::job::job_loop::process_single_job_success_case::ProcessSingleJobSuccessCase;
 use crate::job::job_types::format_conversion::process_single_format_conversion_job::process_single_format_conversion_job;
+use crate::job::job_types::gpt_sovits::process_single_gpt_sovits_job::process_single_gpt_sovits_job;
 use crate::job::job_types::image_generation::process_single_ig_job::process_single_ig_job;
 use crate::job::job_types::lipsync::process_single_lipsync_job::process_single_lipsync_job;
 use crate::job::job_types::mocap::process_single_mc_job::process_single_mc_job;
@@ -254,6 +255,7 @@ fn can_use_new_dispatch(job: &AvailableInferenceJob) -> bool {
  match job.job_type {
    InferenceJobType::VideoRender => true,
    InferenceJobType::LivePortrait => true,
+    InferenceJobType::GptSovits => true,
    _ => false,
  }
 }
@@ -269,7 +271,10 @@ async fn new_dispatch(
    | InferenceJobType::ComfyUi => {
      // NB: These are all comfy workflow jobs too
      process_single_workflow_job(job_dependencies, job).await?
-    }
+    },
+    InferenceJobType::GptSovits => {
+      process_single_gpt_sovits_job(job_dependencies, job).await?
+    },
    _ => {
      return Err(ProcessSingleJobError::InvalidJob(
        anyhow!("invalid job type for dispatch: {:?}", job.job_type)))
@@ -0,0 +1,16 @@
+use errors::AnyhowResult;
+
+use crate::job::job_types::gpt_sovits::gpt_sovits_inference_command::GptSovitsInferenceCommand;
+
+pub struct GptSovitsDependencies {
+  pub inference_command: GptSovitsInferenceCommand,
+}
+
+
+impl GptSovitsDependencies {
+  pub fn setup() -> AnyhowResult<Self> {
+    Ok(Self {
+      inference_command: GptSovitsInferenceCommand::from_env()?,
+    })
+  }
+}
@@ -0,0 +1,252 @@
+use std::collections::HashSet;
+use std::env;
+use std::ffi::OsString;
+use std::fs::File;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+
+use anyhow::anyhow;
+use log::info;
+use once_cell::sync::Lazy;
+use subprocess::{Popen, PopenConfig, Redirection};
+
+use filesys::path_to_string::path_to_string;
+use subprocess_common::command_exit_status::CommandExitStatus;
+use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions};
+
+// These environment vars are not copied over to the subprocess
+// TODO/FIXME(bt, 2023-05-28): This is horrific security!
+static IGNORED_ENVIRONMENT_VARS : Lazy<HashSet<String>> = Lazy::new(|| {
+  let env_var_names= [
+    "MYSQL_URL",
+    "ACCESS_KEY",
+    "SECRET_KEY",
+    "NEWRELIC_API_KEY",
+  ];
+
+  env_var_names.iter()
+    .map(|value| value.to_string())
+    .collect::<HashSet<String>>()
+});
+
+#[derive(Clone)]
+pub struct GptSovitsInferenceCommand {
+  /// Where the code lives
+  pub(crate) gpt_sovits_code_directory: PathBuf,
+
+  /// A single executable script or a much larger bash command.
+  executable_or_command: ExecutableOrCommand,
+
+  /// eg. `source python/bin/activate`
+  maybe_virtual_env_activation_command: Option<String>,
+
+  /// If this is run under Docker (eg. in development), these are the options.
+  maybe_docker_options: Option<DockerOptions>,
+
+  /// If the execution should be ended after a certain point.
+  maybe_execution_timeout: Option<Duration>,
+
+}
+#[derive(Clone)]
+pub enum ExecutableOrCommand {
+  /// Eg. `inference.py`
+  Executable(PathBuf),
+
+  /// Eg. `python3 inference.py`
+  Command(String),
+}
+
+#[derive(Debug)]
+pub struct InferenceArgs<'s> {
+  pub stderr_output_file: &'s Path,
+  pub stdout_output_file: &'s Path,
+
+  pub input_text_file: &'s Path,
+  pub gpt_model_path: &'s Path,
+  pub sovits_model_path: &'s Path,
+  pub reference_audio_path: &'s Path,
+  pub output_audio_directory: &'s Path,
+
+  pub maybe_reference_free: Option<bool>,
+  pub maybe_temperature: Option<f32>,
+  pub maybe_target_language: Option<String>,
+}
+
+impl GptSovitsInferenceCommand {
+  pub fn new(
+    gpt_sovits_code_directory: PathBuf,
+    executable_or_command: ExecutableOrCommand,
+    maybe_virtual_env_activation_command: Option<String>,
+    maybe_docker_options: Option<DockerOptions>,
+    maybe_execution_timeout: Option<Duration>,
+  ) -> Self {
+    Self {
+      gpt_sovits_code_directory,
+      executable_or_command,
+      maybe_virtual_env_activation_command,
+      maybe_docker_options,
+      maybe_execution_timeout,
+    }
+  }
+
+  pub fn from_env() -> anyhow::Result<Self> {
+    let gpt_sovits_code_directory = easyenv::get_env_pathbuf_required("GPT_SOVITS_CODE_DIRECTORY")?;
+
+    let maybe_inference_command = easyenv::get_env_string_optional("GPT_SOVITS_INFERENCE_COMMAND");
+
+    let maybe_inference_executable = easyenv::get_env_pathbuf_optional("GPT_SOVITS_INFERENCE_EXECUTABLE");
+
+    let executable_or_command = match maybe_inference_command {
+      Some(command) => ExecutableOrCommand::Command(command),
+      None => match maybe_inference_executable {
+        Some(executable) => ExecutableOrCommand::Executable(executable),
+        None => return Err(anyhow!("neither command nor executable passed")),
+      },
+    };
+
+    let maybe_virtual_env_activation_command = easyenv::get_env_string_optional("GPT_SOVITS_VENV_ACTIVATION_COMMAND");
+    let maybe_docker_options = easyenv::get_env_string_optional("GPT_SOVITS_DOCKER_OPTIONS")
+    .map(|image_name| {
+      DockerOptions {
+        image_name,
+        maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()),
+        maybe_environment_variables: None,
+        maybe_gpu: Some(DockerGpu::All),
+      }
+    });
+    let maybe_execution_timeout = easyenv::get_env_duration_seconds_optional("GPT_SOVITS_EXECUTION_TIMEOUT");
+
+    Ok(Self {
+      gpt_sovits_code_directory,
+      executable_or_command,
+      maybe_virtual_env_activation_command,
+      maybe_docker_options,
+      maybe_execution_timeout,
+    })
+  }
+
+  pub fn execute_inference(
+    &self,
+    args: InferenceArgs,
+  ) -> CommandExitStatus {
+    match self.do_execute_inference(args) {
+      Ok(exit_status) => exit_status,
+      Err(error) => CommandExitStatus::FailureWithReason { reason: format!("error: {:?}", error) },
+    }
+  }
+
+  pub fn do_execute_inference(
+    &self,
+    args: InferenceArgs,
+  ) -> anyhow::Result<CommandExitStatus> {
+    let mut command = String::new();
+
+
+    command.push_str(&format!("cd {}", path_to_string(&self.gpt_sovits_code_directory)));
+
+    if let Some(venv_command) = self.maybe_virtual_env_activation_command.as_deref() {
+      command.push_str(" && ");
+      command.push_str(venv_command);
+      command.push_str(" ");
+    }
+
+    command.push_str(" && ");
+
+    match self.executable_or_command {
+      ExecutableOrCommand::Executable(ref executable) => {
+        command.push_str(&path_to_string(executable));
+        command.push_str(" infer ");
+      }
+      ExecutableOrCommand::Command(ref cmd) => {
+        command.push_str(cmd);
+        command.push_str(" ");
+      }
+    }
+
+    command.push_str(&format!(" --target_text {}", path_to_string(args.input_text_file)));
+    command.push_str(&format!(" --gpt_model {}", path_to_string(args.gpt_model_path)));
+    command.push_str(&format!(" --sovits_model {}", path_to_string(args.sovits_model_path)));
+    command.push_str(&format!(" --ref_audio {}", path_to_string(args.reference_audio_path)));
+    command.push_str(&format!(" --output_path {}", path_to_string(args.output_audio_directory)));
+
+    if let Some(reference_free) = args.maybe_reference_free {
+      command.push_str(&format!(" --ref_free {}", reference_free));
+    }
+
+    if let Some(temperature) = args.maybe_temperature {
+      command.push_str(&format!(" --temperature {}", temperature));
+    }
+
+    if let Some(target_language) = args.maybe_target_language {
+      command.push_str(&format!(" --target_language {}", target_language));
+    }
+
+    if let Some(maybe_docker_options) = self.maybe_docker_options.as_ref() {
+      command = maybe_docker_options.to_command_string(&command);
+    }
+
+    info!("Running command: {}", command);
+
+    info!("Command: {:?}", command);
+
+    let command_parts = [
+      "bash",
+      "-c",
+      &command
+    ];
+
+    let mut env_vars = Vec::new();
+
+    // Copy all environment variables from the parent process.
+    // This is necessary to send all the kubernetes settings for Nvidia / CUDA.
+    for (env_key, env_value) in env::vars() {
+      if IGNORED_ENVIRONMENT_VARS.contains(&env_key) {
+        continue;
+      }
+      env_vars.push((
+        OsString::from(env_key),
+        OsString::from(env_value),
+      ));
+    }
+
+    let mut config = PopenConfig::default();
+
+    info!("stderr will be written to file: {:?}", args.stderr_output_file.as_os_str());
+
+    let stderr_file = File::create(&args.stderr_output_file)?;
+    config.stderr = Redirection::File(stderr_file);
+
+    if !env_vars.is_empty() {
+      config.env = Some(env_vars);
+    }
+
+    let mut p = Popen::create(&command_parts, config)?;
+
+    info!("Subprocess PID: {:?}", p.pid());
+
+    match self.maybe_execution_timeout {
+      None => {
+        let exit_status = p.wait()?;
+        info!("Subprocess exit status: {:?}", exit_status);
+        Ok(CommandExitStatus::from_exit_status(exit_status))
+      }
+      Some(timeout) => {
+        info!("Executing with timeout: {:?}", &timeout);
+        let exit_status = p.wait_timeout(timeout)?;
+
+        match exit_status {
+          None => {
+            // NB: If the program didn't successfully terminate, kill it.
+            info!("Subprocess didn't end after timeout: {:?}; terminating...", &timeout);
+            let _r = p.terminate()?;
+            Ok(CommandExitStatus::Timeout)
+          }
+          Some(exit_status) => {
+            info!("Subprocess timed wait exit status: {:?}", exit_status);
+            Ok(CommandExitStatus::from_exit_status(exit_status))
+          }
+        }
+      }
+    }
+  }
+}
@@ -0,0 +1,7 @@
+pub mod process_single_gpt_sovits_job;
+mod upload_model;
+mod gpt_sovits_inference_command;
+pub mod gpt_sovits_dependencies;
+mod tts_inference;
+
+mod model_package;
@@ -0,0 +1,79 @@
+use std::collections::HashSet;
+use std::path::{Path, PathBuf};
+
+use anyhow::anyhow;
+use log::{debug, error, info};
+use once_cell::sync::Lazy;
+
+use buckets::public::weight_files::bucket_directory::WeightFileBucketDirectory;
+use buckets::public::weight_files::bucket_file_path::WeightFileBucketPath;
+use cloud_storage::bucket_client::BucketClient;
+use filesys::rename_across_devices::{rename_across_devices, RenameError};
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
+use tokens::tokens::model_weights::ModelWeightToken;
+
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::job::job_types::gpt_sovits::model_package::model_package::GptSovitsPackageFileType;
+use crate::util::filesystem::scoped_temp_dir_creator::ScopedTempDirCreator;
+
+pub async fn download_package(
+  model_weight_token: ModelWeightToken,
+  weights_file_bucket_directory: &WeightFileBucketDirectory,
+  download_directory: &Path,
+  bucket_client: &BucketClient,
+  scoped_tempdir_creator: &ScopedTempDirCreator,
+  force_download: bool,
+) -> Result<(), ProcessSingleJobError> {
+
+  if !download_directory.exists() {
+    std::fs::create_dir_all(&download_directory)
+      .map_err(|err| ProcessSingleJobError::IoError(err))?;
+  }
+
+  let temp_dir_name = format!("model_download_{}", model_weight_token.entropy_suffix());
+  let temp_dir = scoped_tempdir_creator.new_tempdir(&temp_dir_name)
+    .map_err(|e| anyhow!("problem creating tempdir: {:?}", e))?;
+
+  for entry in GptSovitsPackageFileType::all_variants() {
+    let suffix_with_package_identifier = entry.get_expected_package_suffix();
+    let temp_path = temp_dir.path().join(format!("{}{}", model_weight_token.to_string(), suffix_with_package_identifier));
+    let final_download_path = download_directory.join(format!("{}{}", model_weight_token.to_string(), suffix_with_package_identifier));
+
+    if final_download_path.exists() && !force_download {
+      info!("Skipping download of {} because it already exists", final_download_path.display().to_string());
+      continue;
+    }
+
+    if final_download_path.exists() {
+      debug!("Deleting existing file at {:?}", final_download_path.display().to_string());
+      std::fs::remove_file(&final_download_path)
+        .map_err(|err| ProcessSingleJobError::IoError(err))?;
+    }
+
+    let bucket_public_download_path = WeightFileBucketPath::from_object_hash(
+      weights_file_bucket_directory.get_object_hash(),
+      Some("weight_"),
+      Some(&suffix_with_package_identifier),
+    );
+
+    debug!("Downloading to {} from bucket path: {:?}", temp_path.display().to_string(), &bucket_public_download_path.get_full_object_path_str());
+    bucket_client.download_file_to_disk(bucket_public_download_path.get_full_object_path_str(), &temp_path)
+      .await
+      .map_err(|e| {
+        anyhow!("couldn't download {} cloud object to disk: {:?}", &bucket_public_download_path.get_full_object_path_str(), e)
+      })?;
+
+    debug!("Renaming file from {:?} to {:?}!", &temp_path, final_download_path.display().to_string());
+
+    rename_across_devices(&temp_path, final_download_path)
+      .map_err(|err| {
+        safe_delete_temp_directory(&temp_dir);
+        match err {
+          RenameError::StorageFull => ProcessSingleJobError::FilesystemFull,
+          RenameError::IoError(err) => ProcessSingleJobError::from_io_error(err),
+        }})?;
+  }
+
+  safe_delete_temp_directory(&temp_dir);
+  Ok(())
+}
@@ -0,0 +1,2 @@
+pub mod download_package;
+pub mod model_package;
@@ -0,0 +1,102 @@
+use std::collections::{BTreeSet, HashSet};
+
+use once_cell::sync::Lazy;
+
+use buckets::public::weight_files::bucket_file_path::WeightFileBucketPath;
+
+pub const SUFFIX: &str = ".bin";
+
+static ALLOWED_TYPES_GPT : Lazy<HashSet<&'static str>> = Lazy::new(|| {
+  HashSet::from([
+    "ckpt",
+  ])
+});
+
+static ALLOWED_TYPES_SOVITS : Lazy<HashSet<&'static str>> = Lazy::new(|| {
+  HashSet::from([
+    "pth",
+  ])
+});
+
+static ALLOWED_TYPES_REF_AUDIO : Lazy<HashSet<&'static str>> = Lazy::new(|| {
+  HashSet::from([
+    "wav",
+  ])
+});
+
+#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq)]
+pub enum GptSovitsPackageFileType {
+  GptModel,
+  SovitsCheckpoint,
+  ReferenceAudio,
+}
+
+impl GptSovitsPackageFileType {
+  pub fn for_extension(extension: &str) -> Option<Self> {
+    if ALLOWED_TYPES_GPT.contains(extension) {
+      Some(GptSovitsPackageFileType::GptModel)
+    } else if ALLOWED_TYPES_SOVITS.contains(extension) {
+      Some(GptSovitsPackageFileType::SovitsCheckpoint)
+    } else if ALLOWED_TYPES_REF_AUDIO.contains(extension) {
+      Some(GptSovitsPackageFileType::ReferenceAudio)
+    } else {
+      None
+    }
+  }
+
+  pub fn extension_is_allowed(&self, extension: &str) -> bool {
+    match self {
+      GptSovitsPackageFileType::GptModel => ALLOWED_TYPES_GPT.contains(extension),
+      GptSovitsPackageFileType::SovitsCheckpoint => ALLOWED_TYPES_SOVITS.contains(extension),
+      GptSovitsPackageFileType::ReferenceAudio => ALLOWED_TYPES_REF_AUDIO.contains(extension),
+    }
+  }
+
+
+  pub fn all_variants() -> BTreeSet<Self> {
+    // NB: BTreeSet is sorted
+    // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc.
+    BTreeSet::from([
+      Self::GptModel,
+      Self::SovitsCheckpoint,
+      Self::ReferenceAudio,
+    ])
+  }
+
+  pub fn package_identifier(&self) -> &str {
+    match self {
+      GptSovitsPackageFileType::GptModel => "gpt_model",
+      GptSovitsPackageFileType::SovitsCheckpoint => "sovits_checkpoint",
+      GptSovitsPackageFileType::ReferenceAudio => "reference_audio",
+    }
+  }
+
+  pub fn get_expected_package_suffix(&self) -> String {
+    format!(".{}{}", self.package_identifier(), SUFFIX)
+  }
+}
+
+#[derive(Debug)]
+pub enum GptSovitsPackageError {
+  InvalidArchive,
+  InvalidGPTModel(String),
+  InvalidSovitsCheckpoint(String),
+  InvalidReferenceAudio(String),
+  UploadError,
+  TooManyFiles,
+  ExtractionError,
+  FileError,
+}
+
+pub struct GptSovitsPackageFile {
+  pub public_upload_path: WeightFileBucketPath,
+  pub sha256_checksum: String,
+  pub file_size_bytes: u64,
+}
+
+pub struct GptSovitsPackageDetails {
+  pub gpt_model: GptSovitsPackageFile,
+  pub sovits_checkpoint: GptSovitsPackageFile,
+  pub reference_audio: GptSovitsPackageFile,
+}
+
@@ -0,0 +1,33 @@
+use anyhow::anyhow;
+
+use enums::by_table::generic_inference_jobs::inference_job_type::InferenceJobType;
+use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+
+use crate::job::job_loop::job_success_result::JobSuccessResult;
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::job::job_types::gpt_sovits::tts_inference::process_single_gpt_sovits_tts_job::process_single_gpt_sovits_tts_job;
+use crate::job::job_types::gpt_sovits::upload_model::process_gpt_sovits_upload_job::process_gpt_sovits_upload_job;
+use crate::state::job_dependencies::JobDependencies;
+
+pub async fn process_single_gpt_sovits_job(
+  job_dependencies: &JobDependencies,
+  job: &AvailableInferenceJob
+) -> Result<JobSuccessResult, ProcessSingleJobError> {
+  return match job.job_type {
+    InferenceJobType::GptSovits => {
+      match job.maybe_download_url {
+        Some(_) => {
+          let job_success_result = process_gpt_sovits_upload_job(job_dependencies, job).await?;
+          Ok(job_success_result)
+        }
+        None => {
+          let job_success_result = process_single_gpt_sovits_tts_job(job_dependencies, job).await?;
+          Ok(job_success_result)
+        }
+      }
+    }
+    _ => {
+      Err(ProcessSingleJobError::Other(anyhow!("tts model type not set")))
+    }
+  }
+}
@@ -0,0 +1,73 @@
+use anyhow::anyhow;
+
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::InferenceCategoryAbbreviated;
+use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+use tokens::tokens::model_weights::ModelWeightToken;
+
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+
+#[derive(Serialize, Debug)]
+pub struct JobArgs {
+  pub input_text: String,
+  pub gpt_sovits_model: ModelWeightToken,
+  pub reference_free: Option<bool>,
+  pub temperature: Option<f32>,
+  pub target_language: Option<String>,
+}
+
+pub fn check_and_validate_job(job: &AvailableInferenceJob) -> Result<JobArgs, ProcessSingleJobError> {
+  let inference_args = job.maybe_inference_args
+    .as_ref()
+    .map(|args| args.args.as_ref())
+    .flatten();
+
+  let inference_category = job.maybe_inference_args
+    .as_ref()
+    .map(|args| args.inference_category)
+    .flatten();
+
+  match inference_category {
+    Some(InferenceCategoryAbbreviated::GptSovits) => {}, // Valid
+    Some(InferenceCategoryAbbreviated::TextToSpeech) => {},
+    _ => {
+      return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("wrong inference category for job!")));
+    }
+  };
+
+  // let inference_args = match inference_args {
+  //   Some(args) => args,
+  //   None => {
+  //     return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no inference args for job!")));
+  //   }
+  // };
+  //
+  // let inference_args = match inference_args {
+  //   PolymorphicInferenceArgs::Gs(inference_args) => inference_args,
+  //   _ => {
+  //     return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("wrong inner args for job!")));
+  //   }
+  // };
+  //
+  let input_text = match &job.maybe_raw_inference_text {
+    Some(text) => text.clone(),
+    None => {
+      return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no raw inference text for job!")));
+    }
+  };
+
+  let gpt_sovits_model = match &job.maybe_model_token {
+    Some(token) => ModelWeightToken::new_from_str(token),
+    None => {
+      return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no model token for job!")));
+    }
+  };
+
+
+  Ok(JobArgs {
+    input_text: input_text,
+    gpt_sovits_model: gpt_sovits_model,
+    reference_free: None,
+    temperature: None,
+    target_language: None,
+  })
+}
@@ -0,0 +1,2 @@
+pub mod process_single_gpt_sovits_tts_job;
+mod check_and_validate_job;
@@ -0,0 +1,191 @@
+use std::time::Instant;
+
+use anyhow::anyhow;
+use log::info;
+
+use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
+use buckets::public::weight_files::bucket_directory::WeightFileBucketDirectory;
+use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
+use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
+use filesys::check_file_exists::check_file_exists;
+use filesys::file_size::file_size;
+use hashing::sha256::sha256_hash_file::sha256_hash_file;
+use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+use mysql_queries::queries::media_files::create::insert_media_file_from_zero_shot_tts::{insert_media_file_from_zero_shot, InsertArgs};
+use mysql_queries::queries::model_weights::get::get_weight::get_weight_by_token;
+
+use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::job::job_types::gpt_sovits::gpt_sovits_inference_command::InferenceArgs;
+use crate::job::job_types::gpt_sovits::model_package::download_package::download_package;
+use crate::job::job_types::gpt_sovits::model_package::model_package::GptSovitsPackageFileType;
+use crate::job::job_types::gpt_sovits::tts_inference::check_and_validate_job::check_and_validate_job;
+use crate::state::job_dependencies::JobDependencies;
+
+const BUCKET_FILE_PREFIX: &str = "fakeyou_";
+const BUCKET_FILE_EXTENSION: &str = ".wav";
+const MIME_TYPE: &str = "audio/wav";
+
+pub async fn process_single_gpt_sovits_tts_job(
+  job_dependencies: &JobDependencies,
+  job: &AvailableInferenceJob
+) -> Result<JobSuccessResult, ProcessSingleJobError> {
+
+  let mut job_progress_reporter = job_dependencies
+    .clients
+    .job_progress_reporter
+    .new_generic_inference(job.inference_job_token.as_str())
+    .map_err(|e| ProcessSingleJobError::Other(anyhow!(e)))?;
+
+
+  let gpt_sovits_deps = job_dependencies
+    .job
+    .job_specific_dependencies
+    .maybe_gpt_sovits_dependencies
+    .as_ref()
+    .ok_or_else(|| ProcessSingleJobError::JobSystemMisconfiguration(Some("Missing GPT-Sovits dependencies".to_string())))?;
+
+  let job_args = check_and_validate_job(job)?;
+
+  let work_temp_dir = format!("temp_gpt_sovits_tts_{}", job.id.0);
+
+  // NB: TempDir exists until it goes out of scope, at which point it should delete from filesystem.
+  let work_temp_dir = job_dependencies
+    .fs
+    .scoped_temp_dir_creator_for_work
+    .new_tempdir(&work_temp_dir)
+    .map_err(|e| ProcessSingleJobError::from_io_error(e))?;
+
+  let output_dir = work_temp_dir.path().join("output");
+  let output_file_path = output_dir.join("GPTSoVITS_1.wav");
+
+  if !output_dir.exists() {
+    std::fs::create_dir_all(&output_dir)
+      .map_err(|err| ProcessSingleJobError::IoError(err))?;
+  }
+
+  let input_text = job_args.input_text;
+  let gpt_sovits_model = job_args.gpt_sovits_model;
+
+  let model_record = get_weight_by_token(&gpt_sovits_model, false, &job_dependencies.db.mysql_pool)
+    .await?.ok_or_else(|| ProcessSingleJobError::ModelDeleted)?;
+
+  let model_token = model_record.token.clone();
+  let weights_file_bucket_directory = WeightFileBucketDirectory::from_object_hash(&model_record.public_bucket_hash);
+
+  let weights_directory = &job_dependencies.fs.semi_persistent_cache.gpt_sovits_model_path(&model_token.as_str());
+
+  // TODO(KS): Expose this as configuration
+  let force_download = job.attempt_count > 2;
+
+  download_package(
+    gpt_sovits_model,
+    &weights_file_bucket_directory,
+    &weights_directory,
+    &job_dependencies.buckets.public_bucket_client,
+    &job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
+    force_download,
+  ).await?;
+
+  let stderr_output_file = work_temp_dir.path().join("stderr.txt");
+  let stdout_output_file = work_temp_dir.path().join("stdout.txt");
+  let text_input_fs_path = work_temp_dir.path().join("inference_input.txt");
+  std::fs::write(&text_input_fs_path, &input_text)
+    .map_err(|e| ProcessSingleJobError::from_io_error(e))?;
+
+  let gpt_model_path = weights_directory.join(format!("{}{}", model_token.as_str(), GptSovitsPackageFileType::GptModel.get_expected_package_suffix()));
+  let sovits_model_path = weights_directory.join(format!("{}{}", model_token.as_str(), GptSovitsPackageFileType::SovitsCheckpoint.get_expected_package_suffix()));
+  let reference_audio_path = weights_directory.join(format!("{}{}", model_token.as_str(), GptSovitsPackageFileType::ReferenceAudio.get_expected_package_suffix()));
+
+  let inference_start_time = Instant::now();
+  let command_exit_status = gpt_sovits_deps
+    .inference_command
+    .execute_inference(InferenceArgs{
+      stderr_output_file: &stderr_output_file,
+      stdout_output_file: &stdout_output_file,
+      input_text_file: &text_input_fs_path,
+      gpt_model_path: &gpt_model_path,
+      sovits_model_path: &sovits_model_path,
+      reference_audio_path: &reference_audio_path,
+      output_audio_directory: &output_dir,
+      maybe_reference_free: None,
+      maybe_temperature: None,
+      maybe_target_language: Some("english".to_string()),
+    });
+  let inference_duration = Instant::now().duration_since(inference_start_time);
+
+  info!("Inference command exited with status: {:?}", command_exit_status);
+  info!("Inference took duration to complete: {:?}", &inference_duration);
+
+  // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== //
+
+  info!("Checking that output files exist...");
+
+  check_file_exists(&output_file_path).map_err(|e| ProcessSingleJobError::Other(e))?;
+
+  // upload audio to bucket
+  info!("Uploading media ...");
+
+  let result_bucket_location = MediaFileBucketPath::generate_new(
+    Some(BUCKET_FILE_PREFIX),
+    Some(BUCKET_FILE_EXTENSION)
+  );
+
+  let result_bucket_object_pathbuf = result_bucket_location.to_full_object_pathbuf();
+
+  // Finished file path
+  info!("Upload Bucket Path: {:?}", result_bucket_object_pathbuf);
+  info!("Upload File Path: {:?}", &output_file_path);
+
+  job_dependencies.buckets.public_bucket_client
+    .upload_filename_with_content_type(
+      &result_bucket_object_pathbuf,
+      &output_file_path,
+      &MIME_TYPE
+    )
+    .await
+    .map_err(|e| ProcessSingleJobError::Other(e))?;
+  
+  // ==================== UPLOAD AUDIO TO BUCKET ====================
+
+  info!("Calculating sha256...");
+
+  let file_checksum = sha256_hash_file(&output_file_path).map_err(|err| {
+    ProcessSingleJobError::Other(anyhow!("Error hashing file: {:?}", err))
+  })?;
+
+  let file_size_bytes = file_size(&output_file_path).map_err(|err|
+    ProcessSingleJobError::Other(err)
+  )?;
+
+  job_progress_reporter.log_status("done").map_err(|e| ProcessSingleJobError::Other(e))?;
+  let (media_file_token, id) = insert_media_file_from_zero_shot(InsertArgs {
+    pool: &job_dependencies.db.mysql_pool,
+    job: &job,
+    origin_model_type: MediaFileOriginModelType::GptSovits,
+    maybe_mime_type: Some(&MIME_TYPE),
+    file_size_bytes,
+    sha256_checksum: &file_checksum,
+    public_bucket_directory_hash: result_bucket_location.get_object_hash(),
+    maybe_public_bucket_prefix: Some(BUCKET_FILE_PREFIX),
+    maybe_public_bucket_extension: Some(BUCKET_FILE_EXTENSION),
+    is_on_prem: job_dependencies.job.info.container.is_on_prem,
+    worker_hostname: &job_dependencies.job.info.container.hostname,
+    worker_cluster: &job_dependencies.job.info.container.cluster_name,
+  }).await.map_err(|e| ProcessSingleJobError::Other(e))?;
+
+  info!(
+        "Job {:?} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}, Result Token: {}",
+        job.id,
+        id,
+        &media_file_token
+    );
+
+  Ok(JobSuccessResult {
+    maybe_result_entity: Some(ResultEntity {
+      entity_type: InferenceResultType::MediaFile,
+      entity_token: media_file_token.to_string(),
+    }),
+    inference_duration,
+  })
+}
@@ -0,0 +1,246 @@
+use std::io::{BufReader, Cursor, Read};
+use std::path::PathBuf;
+
+use log::{error, info, warn};
+use zip::ZipArchive;
+
+use buckets::public::weight_files::bucket_directory::WeightFileBucketDirectory;
+use buckets::public::weight_files::bucket_file_path::WeightFileBucketPath;
+use cloud_storage::bucket_client::BucketClient;
+use filesys::path_to_string::path_to_string;
+use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
+
+use crate::job::job_types::gpt_sovits::model_package::model_package::{GptSovitsPackageDetails, GptSovitsPackageError, GptSovitsPackageFile, GptSovitsPackageFileType};
+
+pub async fn extract_and_upload_gpt_sovits_package_files(
+  zip_container_file_bytes: &[u8],
+  bucket_client: &BucketClient,
+  weights_file_bucket_directory: &WeightFileBucketDirectory,
+) -> Result<GptSovitsPackageDetails, GptSovitsPackageError> {
+  let cursor = Cursor::new(zip_container_file_bytes);
+  let reader = BufReader::new(cursor);
+  let mut archive = ZipArchive::new(reader)
+    .map_err(|err| {
+      error!("Error reading zip archive: {:?}", err);
+      GptSovitsPackageError::InvalidArchive
+    })?;
+
+  if archive.len() > 255 {
+    return Err(GptSovitsPackageError::TooManyFiles);
+  }
+
+  let mut gpt_model: Option<GptSovitsPackageFile> = None;
+  let mut sovits_checkpoint: Option<GptSovitsPackageFile> = None;
+  let mut reference_audio: Option<GptSovitsPackageFile> = None;
+
+  let entries = get_relevant_zip_entries(&mut archive)?;
+
+  for entry in entries.iter() {
+    info!("Entry: {:?}", entry);
+
+    let enclosed_name = path_to_string(&entry.enclosed_name);
+    let mut file = archive.by_name(&enclosed_name)
+      .map_err(|err| {
+        error!("Problem reading file from archive: {:?}", err);
+        GptSovitsPackageError::InvalidArchive
+      })?;
+
+
+    let mut zip_item_bytes = Vec::new();
+
+    file.read_to_end(&mut zip_item_bytes)
+      .map_err(|err| {
+        error!("Problem reading file from archive: {:?}", err);
+        GptSovitsPackageError::ExtractionError
+      })?;
+
+    match entry.package_type {
+      GptSovitsPackageFileType::GptModel => {
+        let bucket_public_upload_path = WeightFileBucketPath::from_object_hash(
+          weights_file_bucket_directory.get_object_hash(),
+          Some("weight_"),
+          Some(&entry.package_type.get_expected_package_suffix()),
+        );
+        info!("Uploading GPT model to: {}", bucket_public_upload_path.get_full_object_path_str());
+        bucket_client.upload_file(
+          bucket_public_upload_path.get_full_object_path_str(),
+          zip_item_bytes.as_ref())
+          .await
+          .map_err(|e| {
+            warn!("Upload gpt package to bucket error: {:?}", e);
+            GptSovitsPackageError::UploadError
+          })?;
+
+        let hash = sha256_hash_bytes(&zip_item_bytes)
+          .map_err(|io_error| {
+            error!("Problem hashing bytes: {:?}", io_error);
+            GptSovitsPackageError::UploadError
+          })?;
+
+        let file_size_bytes = zip_item_bytes.len();
+
+        gpt_model = Some(GptSovitsPackageFile {
+          public_upload_path: bucket_public_upload_path,
+          sha256_checksum: hash,
+          file_size_bytes: file_size_bytes as u64,
+        });
+      },
+      GptSovitsPackageFileType::SovitsCheckpoint => {
+        let bucket_public_upload_path = WeightFileBucketPath::from_object_hash(
+          weights_file_bucket_directory.get_object_hash(),
+          Some("weight_"),
+          Some(&entry.package_type.get_expected_package_suffix()),
+        );
+        info!("Uploading sovits checkpoint to {:?}", bucket_public_upload_path.get_full_object_path_str());
+        bucket_client.upload_file(
+          bucket_public_upload_path.get_full_object_path_str(),
+          zip_item_bytes.as_ref())
+          .await
+          .map_err(|e| {
+            warn!("Upload sovits package to bucket error: {:?}", e);
+            GptSovitsPackageError::UploadError
+          })?;
+
+        let hash = sha256_hash_bytes(&zip_item_bytes)
+          .map_err(|io_error| {
+            error!("Problem hashing bytes: {:?}", io_error);
+            GptSovitsPackageError::UploadError
+          })?;
+
+        let file_size_bytes = zip_item_bytes.len();
+
+        sovits_checkpoint = Some(GptSovitsPackageFile {
+          public_upload_path: bucket_public_upload_path,
+          sha256_checksum: hash,
+          file_size_bytes: file_size_bytes as u64,
+        });
+      },
+      GptSovitsPackageFileType::ReferenceAudio => {
+        let bucket_public_upload_path = WeightFileBucketPath::from_object_hash(
+          weights_file_bucket_directory.get_object_hash(),
+          Some("weight_"),
+          Some(&entry.package_type.get_expected_package_suffix()),
+        );
+        info!("Uploading reference audio package to {:?}", bucket_public_upload_path.get_full_object_path_str());
+        bucket_client.upload_file(
+          bucket_public_upload_path.get_full_object_path_str(),
+          zip_item_bytes.as_ref())
+          .await
+          .map_err(|e| {
+            warn!("Upload reference audio package to bucket error: {:?}", e);
+            GptSovitsPackageError::UploadError
+          })?;
+
+        let hash = sha256_hash_bytes(&zip_item_bytes)
+          .map_err(|io_error| {
+            error!("Problem hashing bytes: {:?}", io_error);
+            GptSovitsPackageError::UploadError
+          })?;
+
+        let file_size_bytes = zip_item_bytes.len();
+
+        reference_audio = Some(GptSovitsPackageFile {
+          public_upload_path: bucket_public_upload_path,
+          sha256_checksum: hash,
+          file_size_bytes: file_size_bytes as u64,
+        });
+      },
+    }
+  }
+
+  Ok(GptSovitsPackageDetails {
+    gpt_model: gpt_model.ok_or(GptSovitsPackageError::InvalidArchive)?,
+    sovits_checkpoint: sovits_checkpoint.ok_or(GptSovitsPackageError::InvalidArchive)?,
+    reference_audio: reference_audio.ok_or(GptSovitsPackageError::InvalidArchive)?,
+  })
+}
+
+#[derive(Debug,Clone)]
+struct PackageZipEntryDetails {
+  package_type: GptSovitsPackageFileType,
+  enclosed_name: PathBuf,
+  maybe_better_alternative_output_name: String,
+  file_size: u64,
+  is_valid_file_extension: bool,
+}
+
+fn get_relevant_zip_entries(archive: &mut ZipArchive<BufReader<Cursor<&[u8]>>>) -> Result<Vec<PackageZipEntryDetails>, GptSovitsPackageError> {
+  let mut entries: Vec<PackageZipEntryDetails> = Vec::new();
+
+  for i in 0..archive.len() {
+    info!("Reading file {}...", i);
+
+    let file = archive.by_index(i)
+      .map_err(|err| {
+        error!("Problem reading file from archive: {:?}", err);
+        GptSovitsPackageError::InvalidArchive
+      })?;
+
+    let filename = file.name();
+    let filename_lowercase = filename.to_lowercase();
+
+    info!("File {} is {:?} - is file = {}", i, filename, file.is_file());
+    info!("Enclosed name: {:?}", file.enclosed_name());
+
+    if file.is_dir() {
+      info!("Skipping directory: {:?}", filename);
+      continue;
+    }
+
+    if filename_lowercase.starts_with("__macosx/") {
+      info!("Skipping __MACOSX directory entry: {:?}", filename);
+      // Mac users sometimes have a bogus __MACOSX directory, which may double the file count.
+      continue;
+    }
+
+    let enclosed_name = match file.enclosed_name() {
+      None => return Err(GptSovitsPackageError::FileError),
+      Some(name) => name,
+    };
+
+    let extension = enclosed_name.extension()
+      .map(|ext| ext.to_str().unwrap_or(""))
+      .unwrap_or("");
+    
+    info!("Attempting to process file with name {} extension: {}", enclosed_name.display(), extension);
+
+    match GptSovitsPackageFileType::for_extension(extension) {
+      None => {
+        info!("Skipping file with name {} extension: {}", enclosed_name.display(), extension);
+      }
+      Some(package_type) => {
+        if entries.iter().any(|entry| &entry.package_type == &package_type) {
+          return match package_type {
+            GptSovitsPackageFileType::GptModel => {
+              Err(GptSovitsPackageError::InvalidGPTModel("Multiple GPT models found".to_string()))
+            }
+            GptSovitsPackageFileType::SovitsCheckpoint => {
+              Err(GptSovitsPackageError::InvalidSovitsCheckpoint("Multiple Sovits checkpoints found".to_string()))
+            }
+            GptSovitsPackageFileType::ReferenceAudio => {
+              Err(GptSovitsPackageError::InvalidReferenceAudio("Multiple reference audio files found".to_string()))
+            }
+          }
+        }
+        info!("Adding file with name {} extension: {}", enclosed_name.display(), extension);
+        entries.push(PackageZipEntryDetails {
+          enclosed_name: enclosed_name.to_path_buf(),
+          maybe_better_alternative_output_name: package_type.package_identifier().to_string(),
+          file_size: file.size(),
+          is_valid_file_extension: package_type.extension_is_allowed(extension),
+          package_type,
+        });
+      }
+    }
+  }
+
+  if entries.len() < GptSovitsPackageFileType::all_variants().len() {
+    return Err(GptSovitsPackageError::InvalidArchive);
+  }
+
+  for entry in entries.iter() {
+    info!("Entry: {:?}", entry);
+  }
+
+  Ok(entries)
+}
@@ -0,0 +1,26 @@
+use anyhow::anyhow;
+
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
+use mysql_queries::payloads::generic_inference_args::gptsovits_payload::GptSovitsPayload;
+use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::util::extractors::get_polymorphic_args_from_job::get_polymorphic_args_from_job;
+
+pub fn extract_gpt_sovits_payload_from_job(
+  job: &AvailableInferenceJob,
+) -> Result<GptSovitsPayload, ProcessSingleJobError> {
+
+  let polymorphic_args = get_polymorphic_args_from_job(&job)?;
+
+  let some_args = match polymorphic_args {
+    PolymorphicInferenceArgs::Gs(payload) => payload.clone(),
+    _ => {
+      return Err(
+        ProcessSingleJobError::from_anyhow_error(anyhow!("wrong inner args for job!"))
+      );
+    }
+  };
+
+  Ok(some_args)
+}
@@ -0,0 +1,3 @@
+pub mod process_gpt_sovits_upload_job;
+pub mod extract_gpt_sovits_payload_from_job;
+mod extract_and_upload_gpt_sovits_package;
@@ -0,0 +1,175 @@
+use std::time::Duration;
+
+use anyhow::{anyhow, Result};
+use log::{info, warn};
+
+use buckets::public::weight_files::bucket_directory::WeightFileBucketDirectory;
+use buckets::public::weight_files::bucket_file_path::WeightFileBucketPath;
+use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
+use enums::by_table::model_weights::weights_category::WeightsCategory;
+use enums::by_table::model_weights::weights_types::WeightsType;
+use enums::common::visibility::Visibility;
+use filesys::file_exists::file_exists;
+use filesys::file_read_bytes::file_read_bytes;
+use filesys::file_size::file_size;
+use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
+use hashing::sha256::sha256_hash_file::sha256_hash_file;
+use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+use mysql_queries::queries::model_weights::create::create_weight;
+use mysql_queries::queries::model_weights::create::create_weight::CreateModelWeightsArgs;
+use tokens::tokens::model_weights::ModelWeightToken;
+use tokens::tokens::users::UserToken;
+
+use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::job::job_types::gpt_sovits::model_package::model_package::SUFFIX;
+use crate::job::job_types::gpt_sovits::upload_model::extract_and_upload_gpt_sovits_package::extract_and_upload_gpt_sovits_package_files;
+use crate::job::job_types::gpt_sovits::upload_model::extract_gpt_sovits_payload_from_job::extract_gpt_sovits_payload_from_job;
+use crate::state::job_dependencies::JobDependencies;
+
+pub async fn process_gpt_sovits_upload_job(deps: &JobDependencies, job: &AvailableInferenceJob) -> Result<JobSuccessResult, ProcessSingleJobError>{
+  let mysql_pool = &deps.db.mysql_pool;
+
+  let bucket_client = &deps.buckets.public_bucket_client;
+
+  let upload_args = extract_gpt_sovits_payload_from_job(&job)?;
+
+  let title = match upload_args.maybe_title {
+    Some(val) => {
+      val
+    },
+    None => { "".to_string() }
+  };
+
+  let description = match upload_args.maybe_description {
+    Some(val) => {
+      val
+    },
+    None => { "".to_string() }
+  };
+
+  let visibility = match upload_args.creator_visibility {
+    Some(val) => {
+      val
+    },
+    None => { Visibility::Public }
+  };
+
+
+  let file_name = "model.zip";
+
+  let download_script = easyenv::get_env_string_or_default(
+    "DOWNLOAD_SCRIPT",
+    "download_internet_file.py"
+  );
+
+  let creator_ip_address = &job.creator_ip_address;
+
+  let creator_user_token = match &job.maybe_creator_user_token {
+    Some(token) => UserToken::new_from_str(token),
+    None => return Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Creator User Token"))),
+  };
+
+  let download_url = match &job.maybe_download_url {
+    Some(val) => val.to_string(),
+    None => {
+      return Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Download URL")));
+    }
+  };
+
+  if download_url.len() == 0 {
+    return Err(ProcessSingleJobError::InvalidJob(anyhow!("Download URL Too Short")));
+  }
+
+  let google_drive_downloader = GoogleDriveDownloadCommand::new(
+    &download_script, None, None, None);
+
+  info!("Downloading {}", download_url);
+
+  let work_temp_dir = format!("gpt_sovits_upload_{}", job.id.0);
+  let work_temp_dir = deps.fs.scoped_temp_dir_creator_for_work
+    .new_tempdir(&work_temp_dir)
+    .map_err(|e| ProcessSingleJobError::from_io_error(e))?;
+
+  let download_filename = match google_drive_downloader.download_file_with_file_name(
+    &download_url,
+    &work_temp_dir,
+    file_name
+  ).await
+  {
+    Ok(filename) => filename,
+    Err(_e) => return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download"))),
+  };
+
+  let download_file_path = work_temp_dir.path().join(download_filename);
+
+  if file_exists(download_file_path.as_path()) == false {
+    return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download")));
+  }
+
+  info!("File Retrieved at {}", download_file_path.display());
+
+  const PREFIX: Option<&str> = Some("weight_");
+
+  let mut file_bytes = Vec::new();
+  file_bytes = file_read_bytes(&download_file_path)
+    .map_err(|e| ProcessSingleJobError::from_anyhow_error(anyhow!("Processing archive failed")))?;
+
+  let model_weight_token: &ModelWeightToken = &ModelWeightToken::generate();
+  let weight_file_bucket_directory = WeightFileBucketDirectory::from_object_hash(model_weight_token.entropy_suffix());
+
+  // TODO(KS, 22-07-2023): We should decide if these details are worth persisting anywhere
+  let _ = extract_and_upload_gpt_sovits_package_files(&file_bytes, bucket_client, &weight_file_bucket_directory)
+    .await.map_err(|easyenv| {
+    warn!("Failed to extract and verify GPT-SoViTS package: {:?}", easyenv);
+    ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to extract and verify GPT-SoViTS package"))
+  })?;
+
+  let bucket_public_upload_path = WeightFileBucketPath::from_object_hash(
+    weight_file_bucket_directory.get_object_hash(),
+    Some("weight_"),
+    Some(".zip"),
+  );
+
+  bucket_client.upload_filename(
+    &bucket_public_upload_path.to_full_object_pathbuf(),&download_file_path)
+    .await
+    .map_err(|e| ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to upload file")))?;
+
+  let file_size_bytes = file_size(&download_file_path)
+    .map_err(|e| ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to get file size")))?;
+  let file_checksum = sha256_hash_file(&download_file_path)
+    .map_err(|e| ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to process archive checksum")))?;
+
+  let model_weight_token_result = create_weight::create_weight(CreateModelWeightsArgs {
+    token: &model_weight_token,
+    weights_type: WeightsType::GptSoVits,
+    weights_category: WeightsCategory::TextToSpeech,
+    title,
+    maybe_cover_image_media_file_token: job.maybe_cover_image_media_file_token.clone(),
+    maybe_description_markdown: Some(description),
+    maybe_description_rendered_html: None,
+    creator_user_token: Some(&creator_user_token),
+    creator_ip_address,
+    creator_set_visibility: visibility,
+    maybe_last_update_user_token: None,
+    original_download_url: Some(download_url),
+    original_filename: None,
+    // file_size_bytes: metadata.file_size_bytes, // TODO(bt,2024-02-03): We need to migrate the column to be BIGINT
+    file_size_bytes: 0,
+    file_checksum_sha2: file_checksum,
+    public_bucket_hash: bucket_public_upload_path.get_object_hash().to_string(),
+    maybe_public_bucket_prefix: Some(PREFIX.unwrap().to_string()),
+    maybe_public_bucket_extension: Some(".zip".to_string()),
+    version: 0,
+    mysql_pool,
+  }).await?;
+
+  Ok(JobSuccessResult {
+    maybe_result_entity: Some(ResultEntity {
+      entity_type: InferenceResultType::UploadModel,
+      entity_token: model_weight_token_result.to_string(),
+    }),
+    inference_duration: Duration::from_secs(0),
+  })
+}
@@ -7,3 +7,4 @@ pub mod tts;
 pub mod vc;
 pub mod videofilter;
 pub mod workflow;
+pub mod gpt_sovits;
@@ -260,6 +260,7 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
        PolymorphicInferenceArgs::Cu(_) => None,
        PolymorphicInferenceArgs::Es(_) => None,
        PolymorphicInferenceArgs::Lp(_) => None,
+        PolymorphicInferenceArgs::Gs(_) => None,
      })
      .flatten();

@@ -274,6 +275,7 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
        PolymorphicInferenceArgs::Cu(_) => None,
        PolymorphicInferenceArgs::Es(_) => None,
        PolymorphicInferenceArgs::Lp(_) => None,
+        PolymorphicInferenceArgs::Gs(_) => None,
      })
      .flatten();

@@ -181,6 +181,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
        PolymorphicInferenceArgs::Cu(_) => None,
        PolymorphicInferenceArgs::Es(_) => None,
        PolymorphicInferenceArgs::Lp(_) => None,
+        PolymorphicInferenceArgs::Gs(_) => None,
      })
      .flatten()
      .unwrap_or(false);
@@ -196,6 +197,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
        PolymorphicInferenceArgs::Cu(_) => None,
        PolymorphicInferenceArgs::Es(_) => None,
        PolymorphicInferenceArgs::Lp(_) => None,
+        PolymorphicInferenceArgs::Gs(_) => None,
      })
      .flatten();

@@ -51,12 +51,12 @@ use crate::http_server::run_http_server::launch_http_server;
 use crate::job::job_loop::main_loop::main_loop;
 use crate::state::job_dependencies::{BucketDependencies, ClientDependencies, DatabaseDependencies, FileSystemDetails, JobCaches, JobDependencies, JobInstanceInfo, JobSystemControls, JobSystemDependencies};
 use crate::state::job_specific_dependencies::JobSpecificDependencies;
+use crate::state::scoped_job_type_execution::ScopedJobTypeExecution;
+use crate::state::scoped_model_type_execution::ScopedModelTypeExecution;
 use crate::util::filesystem::scoped_temp_dir_creator::ScopedTempDirCreator;
 use crate::util::instrumentation::{init_otel_metrics_pipeline, JobInstrumentLabels};
 use crate::util::instrumentation::JobInstruments;
 use crate::util::model_weights_cache::model_weights_cache_directory::ModelWeightsCacheDirectory;
-use crate::state::scoped_job_type_execution::ScopedJobTypeExecution;
-use crate::state::scoped_model_type_execution::ScopedModelTypeExecution;

 pub mod http_server;
 pub mod job;
@@ -107,6 +107,8 @@ async fn main() -> AnyhowResult<()> {
  let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?;
  let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?;

+  let s3_compatible_endpoint_url = easyenv::get_env_string_or_default("S3_COMPATIBLE_ENDPOINT_URL",
+    "https://storage.googleapis.com");
  let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS",
    Duration::from_secs(60 * 5));

@@ -115,6 +117,7 @@ async fn main() -> AnyhowResult<()> {
    &secret_key,
    &region_name,
    &private_bucket_name,
+    &s3_compatible_endpoint_url,
    None,
    Some(bucket_timeout),
  )?;
@@ -124,6 +127,7 @@ async fn main() -> AnyhowResult<()> {
    &secret_key,
    &region_name,
    &public_bucket_name,
+    &s3_compatible_endpoint_url,
    None,
    Some(bucket_timeout),
  )?;
@@ -367,6 +371,7 @@ fn set_up_directories(job_dependencies: &JobDependencies) -> AnyhowResult<()> {
  fs.semi_persistent_cache.create_tts_pretrained_vocoder_model_path()?;
  fs.semi_persistent_cache.create_tts_synthesizer_model_path()?;
  fs.semi_persistent_cache.create_voice_conversion_model_path()?;
+  fs.semi_persistent_cache.create_gpt_sovits_model_path()?;

  Ok(())
 }
@@ -3,6 +3,7 @@ use enums::by_table::generic_inference_jobs::inference_model_type::InferenceMode
 use errors::AnyhowResult;

 use crate::job::job_types::format_conversion::fbx_to_gltf::dependencies::FbxToGltfDependencies;
+use crate::job::job_types::gpt_sovits::gpt_sovits_dependencies::GptSovitsDependencies;
 use crate::job::job_types::image_generation::sd::stable_diffusion_dependencies::StableDiffusionDependencies;
 use crate::job::job_types::lipsync::sad_talker::sad_talker_dependencies::SadTalkerDependencies;
 use crate::job::job_types::mocap::mocap_net::mocapnet_dependencies::MocapNetDependencies;
@@ -32,6 +33,7 @@ pub struct JobSpecificDependencies {
  pub maybe_comfy_ui_dependencies: Option<ComfyDependencies>,
  pub maybe_convert_fbx_to_gltf_dependencies: Option<FbxToGltfDependencies>,
  pub maybe_convert_bvh_to_workflow_dependencies: Option<RenderEngineSceneToVideoDependencies>,
+  pub maybe_gpt_sovits_dependencies: Option<GptSovitsDependencies>,
 }

 impl JobSpecificDependencies {
@@ -53,6 +55,7 @@ impl JobSpecificDependencies {
    let mut maybe_comfy_ui_dependencies = None;
    let mut maybe_convert_fbx_to_gltf_dependencies = None;
    let mut maybe_convert_bvh_to_workflow_dependencies = None;
+    let mut maybe_gpt_sovits_dependencies = None;

    if scoped_model_type_execution.can_run_job(InferenceModelType::ComfyUi)
        || scoped_job_type_execution.can_run_job(InferenceJobType::LivePortrait)
@@ -63,6 +66,11 @@ impl JobSpecificDependencies {
      maybe_comfy_ui_dependencies = Some(ComfyDependencies::setup().await?);
    }

+    if scoped_job_type_execution.can_run_job(InferenceJobType::GptSovits) {
+      print_with_space("Setting GPT-SoViTS dependencies...");
+      maybe_gpt_sovits_dependencies = Some(GptSovitsDependencies::setup()?);
+    }
+
    if scoped_model_type_execution.can_run_job(InferenceModelType::RvcV2) {
      print_with_space("Setting RVCv2 dependencies...");
      maybe_rvc_v2_dependencies = Some(RvcV2Dependencies::setup()?);
@@ -137,6 +145,7 @@ impl JobSpecificDependencies {
      maybe_comfy_ui_dependencies,
      maybe_convert_fbx_to_gltf_dependencies,
      maybe_convert_bvh_to_workflow_dependencies,
+      maybe_gpt_sovits_dependencies,
    })
  }
 }
@@ -40,7 +40,7 @@ impl ScopedJobTypeExecution {

  pub fn can_run_job(&self, job_type: InferenceJobType) -> bool {
    match self.scoped_types {
-      None => true,
+      None => false,
      Some(ref types) => types.contains(&job_type),
    }
  }
@@ -44,7 +44,7 @@ impl ScopedModelTypeExecution {
  #[deprecated(note="Use ScopedJobTypeExecution instead.")]
  pub fn can_run_job(&self, job_model_type: InferenceModelType) -> bool {
    match self.scoped_types {
-      None => true,
+      None => false,
      Some(ref types) => types.contains(&job_model_type),
    }
  }
@@ -122,6 +122,9 @@ async fn main() -> AnyhowResult<()> {
  let bucket_name = easyenv::get_env_string_required(ENV_BUCKET_NAME)?;
  let bucket_root = easyenv::get_env_string_required(ENV_BUCKET_ROOT)?;

+  let s3_compatible_endpoint_url = easyenv::get_env_string_or_default(
+    "S3_COMPATIBLE_ENDPOINT_URL",
+    "https://storage.googleapis.com");
  let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS",
    Duration::from_secs(60 * 5));

@@ -130,6 +133,7 @@ async fn main() -> AnyhowResult<()> {
    &secret_key,
    &region_name,
    &bucket_name,
+    &s3_compatible_endpoint_url,
    None,
    Some(bucket_timeout),
  )?;
@@ -120,6 +120,9 @@ async fn main() -> AnyhowResult<()> {
  let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?;
  let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?;

+  let s3_compatible_endpoint_url = easyenv::get_env_string_or_default(
+    "S3_COMPATIBLE_ENDPOINT_URL",
+    "https://storage.googleapis.com");
  let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS",
    Duration::from_secs(60 * 5));

@@ -128,6 +131,7 @@ async fn main() -> AnyhowResult<()> {
    &secret_key,
    &region_name,
    &private_bucket_name,
+    &s3_compatible_endpoint_url,
    None,
    Some(bucket_timeout),
  )?;
@@ -137,6 +141,7 @@ async fn main() -> AnyhowResult<()> {
    &secret_key,
    &region_name,
    &public_bucket_name,
+    &s3_compatible_endpoint_url,
    None,
    Some(bucket_timeout),
  )?;
@@ -186,6 +191,7 @@ async fn main() -> AnyhowResult<()> {
  semi_persistent_cache.create_tts_synthesizer_model_path()?;
  semi_persistent_cache.create_tts_pretrained_vocoder_model_path()?;
  semi_persistent_cache.create_custom_vocoder_model_path()?;
+  semi_persistent_cache.create_gpt_sovits_model_path()?;

  let waveglow_vocoder_model_filename = easyenv::get_env_string_or_default(
    "TTS_WAVEGLOW_VOCODER_MODEL_FILENAME", "waveglow.pth");
@@ -150,6 +150,9 @@ async fn main() -> AnyhowResult<()> {
  // Bucket roots
  let bucket_root = easyenv::get_env_string_required(ENV_DOWNLOAD_BUCKET_ROOT)?;

+  let s3_compatible_endpoint_url = easyenv::get_env_string_or_default(
+    "S3_COMPATIBLE_ENDPOINT_URL",
+    "https://storage.googleapis.com");
  let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS",
    Duration::from_secs(60 * 5));

@@ -158,6 +161,7 @@ async fn main() -> AnyhowResult<()> {
    &secret_key,
    &region_name,
    &private_bucket_name,
+    &s3_compatible_endpoint_url,
    None,
    Some(bucket_timeout),
  )?;
@@ -167,6 +171,7 @@ async fn main() -> AnyhowResult<()> {
    &secret_key,
    &region_name,
    &public_bucket_name,
+    &s3_compatible_endpoint_url,
    None,
    Some(bucket_timeout),
  )?;
@@ -144,6 +144,9 @@ async fn main() -> AnyhowResult<()> {
  let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?;
  let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?;

+  let s3_compatible_endpoint_url = easyenv::get_env_string_or_default(
+    "S3_COMPATIBLE_ENDPOINT_URL",
+    "https://storage.googleapis.com");
  let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS",
    Duration::from_secs(60 * 5));

@@ -152,6 +155,7 @@ async fn main() -> AnyhowResult<()> {
    &secret_key,
    &region_name,
    &private_bucket_name,
+    &s3_compatible_endpoint_url,
    None,
    Some(bucket_timeout),
  )?;
@@ -161,6 +165,7 @@ async fn main() -> AnyhowResult<()> {
    &secret_key,
    &region_name,
    &public_bucket_name,
+    &s3_compatible_endpoint_url,
    None,
    Some(bucket_timeout),
  )?;
@@ -0,0 +1,212 @@
+use std::fmt;
+use std::sync::Arc;
+
+use actix_web::{HttpRequest, HttpResponse, web};
+use actix_web::error::ResponseError;
+use actix_web::http::StatusCode;
+use log::{info, log, warn};
+use utoipa::ToSchema;
+
+use config::bad_urls::is_bad_tts_model_download_url;
+use enums::by_table::generic_download_jobs::generic_download_type::GenericDownloadType;
+use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
+use enums::by_table::generic_inference_jobs::inference_job_type::InferenceJobType;
+use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
+use enums::common::visibility::Visibility;
+use http_server_common::request::get_request_ip::get_request_ip;
+use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs};
+use mysql_queries::payloads::generic_inference_args::gptsovits_payload::GptSovitsPayload;
+use mysql_queries::queries::generic_download::web::insert_generic_download_job::{insert_generic_download_job, InsertGenericDownloadJobArgs};
+use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs};
+use tokens::tokens::media_files::MediaFileToken;
+
+use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session;
+use crate::http_server::headers::get_routing_tag_header::get_routing_tag_header;
+use crate::http_server::headers::has_debug_header::has_debug_header;
+use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
+use crate::http_server::validations::validate_model_title::validate_model_title;
+use crate::http_server::web_utils::require_user_session::RequireUserSessionError;
+use crate::http_server::web_utils::require_user_session_using_connection::require_user_session_using_connection;
+use crate::state::server_state::ServerState;
+
+#[derive(Deserialize, ToSchema)]
+pub struct EnqueueGptSovitsModelUploadRequest {
+  pub uuid_idempotency_token: String,
+  pub download_url: String,
+
+  pub maybe_title: Option<String>,
+  pub maybe_cover_image_media_file_token: Option<MediaFileToken>,
+  pub creator_set_visibility: Option<Visibility>,
+}
+
+#[derive(Serialize)]
+pub struct EnqueueGptSovitsModelUploadSuccessResponse {
+  pub success: bool,
+  /// This is how frontend clients can request the job execution status.
+  pub job_token: String,
+}
+
+#[derive(Debug, Serialize)]
+pub enum EnqueueGptSovitsModelUploadError {
+  BadInput(String),
+  NotAuthorized,
+  ServerError,
+  RateLimited,
+}
+
+impl ResponseError for EnqueueGptSovitsModelUploadError {
+  fn status_code(&self) -> StatusCode {
+    match *self {
+      EnqueueGptSovitsModelUploadError::BadInput(_) => StatusCode::BAD_REQUEST,
+      EnqueueGptSovitsModelUploadError::NotAuthorized => StatusCode::UNAUTHORIZED,
+      EnqueueGptSovitsModelUploadError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
+      EnqueueGptSovitsModelUploadError::RateLimited => StatusCode::TOO_MANY_REQUESTS,
+    }
+  }
+
+  fn error_response(&self) -> HttpResponse {
+    serialize_as_json_error(self)
+  }
+}
+
+impl fmt::Display for EnqueueGptSovitsModelUploadError {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{:?}", self)
+  }
+}
+
+pub async fn enqueue_gptsovits_model_upload_handler(
+  http_request: HttpRequest,
+  request: web::Json<EnqueueGptSovitsModelUploadRequest>,
+  server_state: web::Data<Arc<ServerState>>) -> Result<HttpResponse, EnqueueGptSovitsModelUploadError>
+{
+  // ==================== DB ==================== //
+
+  let mut mysql_connection = server_state.mysql_pool
+    .acquire()
+    .await
+    .map_err(|err| {
+      warn!("MySql pool error: {:?}", err);
+      EnqueueGptSovitsModelUploadError::ServerError
+    })?;
+
+  // ==================== USER SESSION ==================== //
+
+  let maybe_avt_token = server_state.avt_cookie_manager
+    .get_avt_token_from_request(&http_request);
+
+  let user_session = require_user_session_using_connection(
+    &http_request,
+    &server_state.session_checker,
+    &mut mysql_connection)
+    .await
+    .map_err(|err| match err {
+      RequireUserSessionError::ServerError => EnqueueGptSovitsModelUploadError::ServerError,
+      RequireUserSessionError::NotAuthorized => EnqueueGptSovitsModelUploadError::NotAuthorized,
+    })?;
+
+  // ==================== PAID PLAN + PRIORITY ==================== //
+
+  // TODO: Plan should handle "first anonymous use" and "investor" cases.
+  let plan = get_correct_plan_for_session(server_state.server_environment, Some(&user_session));
+
+  // TODO: Separate priority for animation.
+  let priority_level = plan.web_vc_base_priority_level();
+
+  let is_staff = user_session.role.can_ban_users;
+
+  // ==================== DEBUG MODE + ROUTING TAG ==================== //
+
+  let is_debug_request = has_debug_header(&http_request);
+
+  let maybe_routing_tag= get_routing_tag_header(&http_request);
+  if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) {
+    return Err(EnqueueGptSovitsModelUploadError::RateLimited);
+  }
+
+
+
+  let ip_address = get_request_ip(&http_request);
+  let uuid = request.uuid_idempotency_token.to_string();
+  let title = request.maybe_title.clone();
+  let download_url = request.download_url.trim().to_string();
+
+  let creator_set_visibility = request.creator_set_visibility
+    .unwrap_or(Visibility::Public);
+
+  if let Err(reason) = validate_idempotency_token_format(&uuid) {
+    return Err(EnqueueGptSovitsModelUploadError::BadInput(reason));
+  }
+
+  // if let Err(reason) = validate_model_title(&title) {
+  //   return Err(EnqueueGptSovitsModelUploadError::BadInput(reason));
+  // }
+
+  // match is_bad_tts_model_download_url(&download_url) {
+  //   Ok(false) => {} // Ok case
+  //   Ok(true) => {
+  //     return Err(EnqueueGptSovitsModelUploadError::BadInput("Bad model download URL".to_string()));
+  //   }
+  //   Err(err) => {
+  //     warn!("Error parsing url: {:?}", err);
+  //     return Err(EnqueueGptSovitsModelUploadError::BadInput("Bad model download URL".to_string()));
+  //   }
+  // }
+
+  let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
+    uuid_idempotency_token: &request.uuid_idempotency_token,
+    job_type: InferenceJobType::GptSovits,
+    inference_category: InferenceCategory::DeprecatedField,
+    maybe_model_type: None,
+    maybe_model_token: None,
+    maybe_input_source_token: None,
+    maybe_input_source_token_type: None,
+    maybe_download_url: Some(&download_url),
+    maybe_cover_image_media_file_token: request.maybe_cover_image_media_file_token.as_ref(),
+    maybe_raw_inference_text: None,
+    maybe_max_duration_seconds: None,
+    maybe_inference_args: Some(GenericInferenceArgs {
+      inference_category: Some(InferenceCategoryAbbreviated::GptSovits),
+      args: Some(PolymorphicInferenceArgs::Gs(
+        GptSovitsPayload {
+          maybe_title: title,
+          maybe_description: None,
+          creator_visibility: Some(creator_set_visibility),
+        })
+      ),
+    }),
+
+    maybe_creator_user_token: Some(&user_session.user_token_typed),
+    maybe_avt_token: maybe_avt_token.as_ref(),
+
+    creator_ip_address: &ip_address,
+    creator_set_visibility: creator_set_visibility,
+    priority_level,
+    requires_keepalive: false, //reverse ...  TODO fix this. we set it base on account is premium or not ...
+    is_debug_request,
+    maybe_routing_tag: maybe_routing_tag.as_deref(),
+    mysql_pool: &server_state.mysql_pool,
+  }).await;
+
+  let job_token = match query_result {
+    Ok((job_token, _id)) => job_token,
+    Err(err) => {
+      warn!("New generic inference job creation DB error: {:?}", err);
+      return Err(EnqueueGptSovitsModelUploadError::ServerError);
+    }
+  };
+
+
+  let response = EnqueueGptSovitsModelUploadSuccessResponse {
+    success: true,
+    job_token: job_token.to_string(),
+  };
+
+  let body = serde_json::to_string(&response)
+    .map_err(|e| EnqueueGptSovitsModelUploadError::ServerError)?;
+
+  Ok(HttpResponse::Ok()
+    .content_type("application/json")
+    .body(body))
+}
@@ -30,13 +30,13 @@ use redis_common::redis_keys::RedisKeys;
 use tokens::tokens::users::UserToken;
 use tts_common::priority::{FAKEYOU_DEFAULT_VALID_API_TOKEN_PRIORITY_LEVEL, FAKEYOU_INVESTOR_PRIORITY_LEVEL};
 use user_input_common::check_for_slurs::contains_slurs;
-use crate::http_server::session::lookup::user_session_extended::UserSessionExtended;

 use crate::configs::app_startup::username_set::UsernameSet;
 use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session;
 use crate::configs::plans::plan::Plan;
 use crate::http_server::endpoints::investor_demo::demo_cookie::request_has_demo_cookie;
 use crate::http_server::endpoints::tts::enqueue_infer_tts_handler::get_tts_model_with_caching::get_tts_model_with_caching;
+use crate::http_server::session::lookup::user_session_extended::UserSessionExtended;
 use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
 use crate::state::server_state::ServerState;

@@ -383,7 +383,6 @@ pub async fn enqueue_infer_tts_handler(

  // ==================== ENQUEUE APPROPRIATE TTS TYPE ==================== //

-  let model_type = InferenceModelType::Tacotron2;

  let max_duration_seconds = plan.tts_max_duration_seconds();

@@ -395,12 +394,16 @@ pub async fn enqueue_infer_tts_handler(
      .map(|token| UserToken::new_from_str(token));

  let maybe_avt_token = server_state.avt_cookie_manager.get_avt_token_from_request(&http_request);
+  let maybe_model_type = match tts_model.job_type() {
+    InferenceJobType::GptSovits => None,
+    _ => Some(InferenceModelType::Tacotron2)
+  };

  let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
    uuid_idempotency_token: &request.uuid_idempotency_token,
-    job_type: InferenceJobType::Tacotron2,
+    job_type: tts_model.job_type(),
    inference_category: InferenceCategory::TextToSpeech,
-    maybe_model_type: Some(model_type),
+    maybe_model_type: maybe_model_type,
    maybe_model_token: Some(request.tts_model_token.as_str()),
    maybe_input_source_token: None, // NB: TTS doesn't have input media
    maybe_input_source_token_type: None, // NB: TTS doesn't have input media
@@ -13,4 +13,5 @@ pub mod get_tts_upload_model_job_status;
 pub mod list_tts_models;
 pub mod list_user_tts_inference_results;
 pub mod list_user_tts_models;
-pub mod search_tts_models_handler;
+pub mod search_tts_models_handler;
+pub mod enqueue_gptsovits_model_upload;
@@ -65,6 +65,7 @@ use crate::http_server::endpoints::tts::delete_tts_model::delete_tts_model_handl
 use crate::http_server::endpoints::tts::delete_tts_result::delete_tts_inference_result_handler;
 use crate::http_server::endpoints::tts::edit_tts_model::edit_tts_model_handler;
 use crate::http_server::endpoints::tts::edit_tts_result::edit_tts_inference_result_handler;
+use crate::http_server::endpoints::tts::enqueue_gptsovits_model_upload::enqueue_gptsovits_model_upload_handler;
 use crate::http_server::endpoints::tts::enqueue_infer_tts_handler::enqueue_infer_tts_handler::enqueue_infer_tts_handler;
 use crate::http_server::endpoints::tts::enqueue_upload_tts_model::upload_tts_model_handler;
 use crate::http_server::endpoints::tts::get_pending_tts_inference_job_count::get_pending_tts_inference_job_count_handler;
@@ -338,6 +339,11 @@ fn add_tts_routes<T, B> (app: App<T>) -> App<T>
            .route(web::post().to(edit_tts_model_handler))
            .route(web::head().to(|| HttpResponse::Ok()))
      )
+      .service(
+        web::resource("/model/gptsovits/upload")
+            .route(web::post().to(enqueue_gptsovits_model_upload_handler))
+            .route(web::head().to(|| HttpResponse::Ok()))
+      )
      .service(
        web::resource("/result/{token}")
            .route(web::get().to(get_tts_inference_result_handler))
@@ -202,6 +202,7 @@ async fn main() -> AnyhowResult<()> {
  // Bucket roots
  let audio_uploads_bucket_root= easyenv::get_env_string_required(ENV_AUDIO_UPLOADS_BUCKET_ROOT)?;

+  let s3_compatible_endpoint_url = easyenv::get_env_string_or_default("S3_COMPATIBLE_ENDPOINT_URL", "https://storage.googleapis.com");
  let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS",
    Duration::from_secs(60 * 5));

@@ -210,6 +211,7 @@ async fn main() -> AnyhowResult<()> {
    &secret_key,
    &region_name,
    &private_bucket_name,
+    &s3_compatible_endpoint_url,
    None,
    Some(bucket_timeout),
  )?;
@@ -219,6 +221,7 @@ async fn main() -> AnyhowResult<()> {
    &secret_key,
    &region_name,
    &public_bucket_name,
+    &s3_compatible_endpoint_url,
    None,
    Some(bucket_timeout),
  )?;
Author	SHA1	Message	Date
Kasisnu	7f8d1f113b	funky dev setup	2024-07-24 16:17:50 +05:30
Kasisnu	a2c1ae7c9b	let existing tts enqueue route accept gptsovits	2024-07-22 19:35:01 +05:30
Kasisnu	f47b729a46	gpt-sovits: inference works	2024-07-22 18:21:36 +05:30
Kasisnu	4f506f05cd	gptsovits: working model upload inference job	2024-07-22 18:21:36 +05:30
Kasisnu	a7d688d750	wip: super basic gptsovits	2024-07-22 18:21:36 +05:30