Minor update

Sad Talker worker, file uploads, etc.

.github/workflows/docker-publish-cpu.yml

@@ -19,7 +19,9 @@ jobs:
   # See also https://evilmartians.com/chronicles/build-images-on-github-actions-with-docker-layer-caching
   build-docker-image:
     name: "Build Docker Image"
-    runs-on: ubuntu-22.04 # runs-on: ubuntu-latest
+    #runs-on: ubuntu-22.04 # runs-on: ubuntu-latest
+    #runs-on: github-ubuntu-16core # NB: Premium GitHub runner with more cores
+    runs-on: github-ubuntu-64core # NB: Premium GitHub runner with more cores
     if: github.event_name == 'push'
     permissions:
       packages: write

.github/workflows/docker-publish-gpu.yml

@@ -19,7 +19,9 @@ jobs:
   # See also https://evilmartians.com/chronicles/build-images-on-github-actions-with-docker-layer-caching
   build-docker-image:
     name: "Build Docker Image"
-    runs-on: ubuntu-22.04 # runs-on: ubuntu-latest
+    #runs-on: ubuntu-22.04 # runs-on: ubuntu-latest
+    #runs-on: github-ubuntu-16core # NB: Premium GitHub runner with more cores
+    runs-on: github-ubuntu-64core # NB: Premium GitHub runner with more cores
     if: github.event_name == 'push'
     permissions:
       packages: write

.idea/dictionaries/bt.xml

@@ -5,10 +5,12 @@
       <w>backfill</w>
       <w>cashapp</w>
       <w>cheermote</w>
+      <w>cidrs</w>
       <w>crockford</w>
       <w>culted</w>
       <w>dalle</w>
       <w>demuxer</w>
+      <w>dereferencing</w>
       <w>downvote</w>
       <w>fakeyou</w>
       <w>ffprobe</w>
@@ -19,6 +21,7 @@
       <w>hifigan</w>
       <w>hparams</w>
       <w>huggingface</w>
+      <w>lipsync</w>
       <w>pathbuf</w>
       <w>replayable</w>
       <w>retryably</w>

Cargo.toml

@@ -16,7 +16,7 @@ members = [
   # Shared libraries
   "crates/lib/actix_helpers",
   "crates/lib/api_clients/cloud_storage",
-  "crates/lib/api_clients/fakeyou_client",
+  #"crates/lib/api_clients/fakeyou_client",
   "crates/lib/api_clients/google_drive_common",
   "crates/lib/api_clients/twitch_common",
   "crates/lib/bootstrap",
@@ -30,6 +30,7 @@ members = [
   "crates/lib/deprecated/http_server_common", # TODO(bt, 2023-03-20): Move useful pieces to actix_helpers
   "crates/lib/envvar",
   "crates/lib/errors",
+  "crates/lib/files/archives",
   "crates/lib/files/filesys",
   "crates/lib/files/hashing",
   "crates/lib/files/media",
@@ -39,7 +40,7 @@ members = [
   "crates/lib/mysql_queries",
   "crates/lib/redis_common",
   "crates/lib/server_environment",
-  "crates/lib/sqlite_queries",
+  #"crates/lib/sqlite_queries",
   "crates/lib/sqlx_mysql_helpers",
   "crates/lib/subprocess_common",
   "crates/lib/tts_common",
@@ -48,7 +49,7 @@ members = [
   "crates/lib/web_scrapers",
 
   # Programs: Desktop apps
-  "crates/programs/aichatbot-sidecar",
+  #"crates/programs/aichatbot-sidecar",
 
   # Services: API servers, jobs, workers, etc.
   "crates/service/job/analytics_job",
@@ -60,7 +61,6 @@ members = [
   "crates/service/job/w2l_inference_job",
   "crates/service/other/twitch_pubsub_subscriber",
   "crates/service/web/dummy_service",
-  "crates/service/web/image_generation_service",
   "crates/service/web/storyteller_web",
   "crates/service/web/websocket_gateway",
 
@@ -68,4 +68,7 @@ members = [
   "crates/service/plugins/billing",
   "crates/service/plugins/inference_workloads",
   "crates/service/plugins/users",
+
+  # Vendored code
+  "crates/vendor/newrelic-telemetry-sdk-rust",
 ]

Cranky.toml

@@ -0,0 +1,58 @@
+# Persist clippy check configurations:
+# https://github.com/ericseppanen/cargo-cranky 
+# Clippy doesn't allow this (yet):
+# https://github.com/rust-lang/rust-clippy/issues/6625
+deny = [
+  # Rust
+  "unsafe_code", # We may remove this if needed
+
+  # Clippy - "serious" lints
+  "clippy::borrow_interior_mutable_const", # Seems dangerous to allow this. Eg. `const F = Lazy::new()`;
+
+  # Clippy - bad math
+  #"clippy::cast_possible_truncation", # Seems dangerous, eg f64 to u64.
+  #"clippy::cast_possible_wrap",
+
+  # Clippy - pointer safety / readability / correctness
+  "clippy::as_ptr_cast_mut",
+  "clippy::borrow_as_ptr",
+  "clippy::borrow_deref_ref",
+
+  # Clippy - performance
+  "clippy::cmp_owned",
+  "clippy::useless_vec",
+  "clippy::format_in_format_args",
+  #"clippy::redundant_clone",
+  #"clippy::unnecessary_to_owned",
+  #"clippy::vec_init_then_push",
+
+  # Clippy - "ergonomic" / "usage" lints
+  "clippy::option_option", # Lint: Don't ever Option<Option<T>>! Enums instead.
+  "clippy::zero_sized_map_values", # Lint: use a HashSet instead of HashMap<T, ()>.
+
+  # Clippy - "aesthetic" / "style" lints
+  "clippy::clone_double_ref", # Lint: feels safer/hygienic to disallow this, but I'm not strongly opposed.
+  "clippy::double_comparisons", # Lint: readability
+  "clippy::empty_structs_with_brackets", # Lint: {} feels less aesthetic than ;
+  "clippy::implicit_clone", # Lint: Recommendation here seems more idiomatic.
+  "clippy::needless_return", # Lint: not the worst offender, but let's try to keep unnecessary returns out.
+  "clippy::redundant_field_names", # Lint: Duplicate names is a burden when reading: you wonder why they're dupes.
+  "clippy::redundant_static_lifetimes", # Lint: feels more modern to elide static lifetimes
+
+  # Possible:
+  # "clippy::ptr_arg",
+  # "clippy::too_many_lines" (note: this is configurable)
+]
+
+warn = [
+]
+
+allow = [
+  # Rust
+  "deprecated", # Other systems can handle deprecated code instead of the linter
+  "missing_docs", # Not everything needs documentation. Too much documentation is bad. Literate code.
+
+  # Clippy
+  "clippy::if_same_then_else", # There are cases this is clearer! Branches may have notes / considerations.
+  "clippy::single_char_pattern", # This is like punishing trailing commas. Chars are dumb.
+]

README.md

@@ -43,4 +43,7 @@ Our Rust monorepo, which serves both our API surface area and our downloadable c
                 .'        /"'          |  "'   '_                      One Rust Monorepo at a time
                /_|.-'\ ,".             '.'`__'-( \
                  / ,"'"\,'               `/  `-.|" 
-```
+```
+
+- [documentation](./_develop/docs)
+- [tools, misc](./_develop)

_develop/docs/README.md

@@ -0,0 +1,52 @@
+README
+======
+
+This directory contains a collection of (hopefully organized and up to date) documentation on 
+running the various backend pieces, development setup, production maintenance, etc. Check out 
+all the markdown documents in this directory tree for information organized by topic.
+
+Development / Setup
+-------------------
+
+- [Most of the general documents for cross-cutting setup of FakeYou.com live here.](./local_development)
+- Some per-application documents (such as for `aichatbot-sidecar`) live as READMEs under that particular 
+  application's code directory, eg. `/crates/{appCategory}/{appName}/README.md` (These are irrelevant 
+  for most folks.)
+
+Web API Overview
+----------------
+
+### FakeYou.com
+
+* [FakeYou.com](https://fakeyou.com) is powered by the following applications:
+    * HTTP APIs:
+        * `storyteller-web` - HTTP API + database monolith for FakeYou
+    * Asynchronous Jobs:
+        * `download-job` - Handle async download of ML models (RVC, SVC, etc.)
+        * `inference-job` - Run async ML model inference (RVC, SVC, SadTalker, etc.) 
+                            This will handle most model inference going forward.
+        * `tts-download-job` - (*deprecated*) Async download of Tacotron2 models only
+        * `tts-inference-job` - (*deprecated*) Async TTS inference of Tacotron2 models only
+        * `w2l-download-job` - (*deprecated*) Async download of videos and images
+        * `w2l-inference-job` - (*deprecated*) Async W2L inference
+
+### PowerStream
+
+* [PowerStream](https://power.stream) (deprecated) is powered by the following applications:
+    * HTTP APIs:
+        * `storyteller-web` - HTTP + database monolith for FakeYou ***and*** PowerStream.
+    * Twitch API integration:
+        * `twitch-pubsub-subscriber` - Dynamically subscribes to Twitch PubSub for "active" users
+    * Websocket events:
+        * `websocket-gateway` - Websocket for users' browsers to pull down enriched Twitch events.
+
+TODO
+----
+
+Notes / TODOs:
+
+* Examples for good Actix+Sqlx Tests:
+  https://stackoverflow.com/questions/65370752/how-do-i-create-an-actix-web-server-that-accepts-both-sqlx-database-pools-and-tr
+
+* Actix/sqlx runtime compat:
+  https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734

_develop/docs/aichatbot_sidecar.md

@@ -1,38 +0,0 @@
-AI Chatbot Sidecar
-==================
-
-This is the first app that doesn't run as a part of our SoA API architecture. (Though it is written such that we 
-could port it.) This app is responsible for generating all the content that goes into our streaming AI-powered TV
-programs.
-
-### Windows Installation
-
-The application uses sqlite, which is tricky to install on Windows.
-
-1. Put together an sqlite build directory containing the sources and dll file. One is checked in under this monorepo 
-   in `./windows/sqlite` at the time of this writing, though that may change in the future.
-
-
-2. Add environment variables for the sqlite build directory (see https://github.com/diesel-rs/diesel/issues/487)
-
-    * PATH (append the sqlite build directory to the end)
-    * SQLITE3_LIB_DIR (a new environment variable; set it to the sqllite build path)
-
-
-3. Build the `sqlite3.lib` file with the following:
-
-   ```
-   lib /def:sqlite3.def /out:sqlite3.lib /MACHINE:amd64 
-   ```
-   
-   This must be done in "Developer Command Prompt for VS 2022" (or similar), not `cmd` or `PowerShell`.
-
-
-4. Run the diesel_cli tool build  
-   ```
-   cargo install diesel_cli --no-default-features --features sqlite
-   ```
-
-   If it still complains about the `sqlite3.lib` file not being found, be sure to double check that `echo %PATH%` shows 
-   the updated PATH from the steps above.
-

_develop/docs/cheat_sheet/migrations.md

@@ -0,0 +1,43 @@
+Migrations guide
+================
+
+## Diesel Migrations
+
+To run pending migrations, execute the following:
+
+```bash
+diesel migration run
+```
+
+To revert the last 3 migrations:
+
+```bash
+diesel migration revert -n 3
+```
+
+To reset the entire database (drop, migrate), run:
+
+```bash
+diesel database reset
+```
+
+## Server Query Codegen
+
+We use SQLx instead of Diesel in the production server. It's typesafe
+SQL instead of an ORM like Diesel.
+
+SQLx connects to a database to derive type information, but obviously
+cannot do this for builds in CI. In order to cache the types, we build
+and check in a cache file (necessary for builds):
+
+```bash
+SQLX_OFFLINE=true cargo sqlx prepare
+```
+
+Now that we have multiple binaries, it's required to include all the queries in the main
+binary so we can generate the cached queries as a single target. That's then executed
+with:
+
+```bash
+SQLX_OFFLINE=true cargo sqlx prepare -- --bin storyteller-web
+```

_develop/docs/local_development/general_setup.md

@@ -0,0 +1,207 @@
+FakeYou Development Setup (bare metal)
+======================================
+
+This document describes how to run the FakeYou infrastructure on your development 
+machine's bare metal (no containers).
+
+Setup instructions and development were tested and known to work on the following
+platforms:
+
+- Mac OS (Apple M2 silicon)
+- Ubuntu 22.04
+
+The applications (binary targets) you might be interested in running include, but 
+are not limited to:
+
+- `storyteller-web`, the HTTP API server
+- `download-job`, which downloads models from the web and stores them in GCS
+- `inference-job`, which downloads models from GCS and runs inference with them
+
+Setup Instructions
+------------------
+
+### (1) Install Rust
+
+Install Rust [using this guide](https://www.rust-lang.org/learn/get-started). If it asks, 
+you'll want "stable" Rust, not "nightly" Rust. If it doesn't ask, it defaults to "stable".
+
+### (2) Install the necessary libraries
+
+Mac: (Lost the list, but it should match Linux. Install Homebrew.)
+
+Ubuntu 22.04:
+
+```bash
+sudo apt install \ 
+  jq \
+  libmysqlclient-dev \
+  libsqlite3-dev \
+  pkgconf
+```
+
+(Note: jq is for combining mysql codegen outputs; pkgconf is if using openssl instead of rustls.)
+
+### (3) Install MySQL server
+
+Mac:
+
+```bash
+brew install mysql
+```
+
+If `mysql -uroot` fails, reboot the machine:
+
+```bash
+sudo reboot now
+```
+
+Ubuntu 22.04:
+
+```bash
+sudo apt install mysql-server
+```
+
+
+### (4) Install a `storyteller` user and table in MySQL
+
+Connect to mysql:
+
+```bash
+# If the following command asks for a password, the password is typically "root"
+sudo mysql -u root -p
+```
+
+Once in MySQL, run the following:
+
+```mysql
+use mysql;
+CREATE DATABASE storyteller;
+CREATE USER 'storyteller'@'localhost' IDENTIFIED BY 'password';
+GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';
+```
+
+Then verify access with `./dev_mysql_connect.sh`
+
+### (5) Install Diesel CLI (for MySQL migrations):
+
+```bash
+cargo install diesel_cli \
+  --no-default-features \
+  --features mysql,sqlite
+```
+
+As of this writing, Mac [has some issues with Diesel CLI](https://github.com/diesel-rs/diesel/issues/2605)
+and requires a few extra dependencies to be installed:
+
+```bash
+brew install libpq
+```
+
+### (6) Run the pending database migrations:
+
+```bash
+diesel migration run
+```
+
+You might get a scary message about `"Encountered unknown type for Mysql: enum"` -- you can ignore this 
+error (see below).
+
+### (7) (optional) Install the SQLx CLI (if doing database development)
+
+```bash
+cargo install sqlx-cli --no-default-features --features rustls,mysql,sqlite
+```
+
+We'll be using diesel to manage the migrations, but sqlx within the app to actually perform queries.
+Diesel is an immature ORM, which isn't a good tech bet, so we use sqlx as at-compile-time
+typesafe SQL.
+
+### (8) Install hosts file:
+
+(This should be the same on Linux and Mac, but might differ for Windows.)
+
+If you're developing against the frontend, it'll target development domains (eg. `dev.fakeyou.com`) instead 
+of `127.0.0.1` or `localhost`. You can make your machine route domains to localhost by editing your hosts 
+file (located at `/etc/hosts`) to include the following configuration lines:
+
+```
+127.0.0.1    dev.fakeyou.com
+127.0.0.1    api.dev.fakeyou.com
+127.0.0.1    devproxy.fakeyou.com
+
+127.0.0.1    dev.storyteller.ai
+127.0.0.1    api.dev.storyteller.ai
+127.0.0.1    devproxy.storyteller.ai
+```
+
+### (9a) Install Redis
+
+```
+sudo apt install redis
+```
+
+### (9b) Run one or more applications:
+DO NOT CHECK THESE TWO FILES INTO THE REPO
+Ask Brandon for the .env-secrets and .storteller-web.development-secrets.env and place it in the root of the folder.
+
+To start the HTTP API server,
+
+```bash
+cargo run --bin storyteller-web
+```
+Note that this compiles and runs the "development" binary. It's faster and easier to debug than the
+optimized "production" build. To build a fully optimized production release,
+run `cargo build --release --bin storyteller-web` . Note that this will take much longer.
+
+DO NOT CHECK THESE TWO FILES INTO THE REPO
+
+Ask Brandon for the .download-job.development-secrets.env and place it in
+crates/service/job/download_job/config
+
+Ask Brandon for the .inference-job.development-secrets.env and place it in
+crates/service/job/inference_job/config
+
+To download some ML models, run:
+
+```bash
+cargo run --bin download-job
+cargo run --bin tts-download-job
+```
+
+If you want to run both the HTTP API and the jobs, you'll need to run both processes in different
+terminals or tmux sessions.
+
+Note also that the configurations pointing to the ML monorepo must be set up for each application.
+Additionally, some development secrets may be needed (ask the team to share).
+
+To execute ML inference, run:
+
+```bash
+cargo run --bin inference-job
+cargo run --bin tts-inference-job
+```
+
+Again, note that the configurations pointing to the ML monorepo must be set up for each application.
+Additionally, some development secrets may be needed (ask the team to share).
+
+Solutions to Common Setup Problems 
+-----------------------------------
+
+### [Fix] Sqlx Error when Performing Schema Migrations
+
+You might get this error message during migration,
+
+```
+Encountered unknown type for Mysql: enum
+thread 'main' panicked at 'internal error: entered unreachable code: Mysql only supports a closed set of types.
+                         If you ever see this error message please open an issue at https://github.com/diesel-rs/diesel 
+                         containing a dump of your schema definition.', diesel_cli/src/print_schema.rs:310:17
+note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
+```
+
+I haven't found the cause (it doesn't happen on newer installs), but the migrations appear to work regardless 
+of this error message. You can essentially ignore it.
+
+### [Fix] Can't connect to local MySQL after install on Ubuntu
+
+- If MySql in local dev can't be connected to, [reset the accounts](https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux).

_develop/docs/local_development/local_development_k8s.md

@@ -1,5 +1,7 @@
 # Local Kubernetes environment for standing up the APIs and their dependencies
 
+(These are Paul's notes.)
+
 - Install minikube, kubectl, docker, and musl from your OS package manager
 - `rustup target add x86_64-unknown-linux-musl`
 - `cargo build --release --bin storyteller-web --target=x86_64-unknown-linux-musl`

_develop/docs/local_development/local_development_nginx_proxy.md

@@ -0,0 +1,56 @@
+Local Development nginx proxy
+=============================
+
+It isn't necessary to use nginx for development, but it can be nice. Nginx allows for a more 
+sophisticated testing setup that makes cookie and CORS issues less of a pain.
+
+You can use nginx to serve (locally-signed) SSL, redirect traffic to different ports, or run 
+multiple copies of the app side by side.
+
+### Hosts file
+
+Set the following configs in your `/etc/hosts` file (as you would probably do even without nginx),
+
+```
+127.0.0.1    dev.fakeyou.com
+127.0.0.1    api.dev.fakeyou.com
+127.0.0.1    devproxy.fakeyou.com
+
+127.0.0.1    dev.storyteller.ai
+127.0.0.1    api.dev.storyteller.ai
+127.0.0.1    devproxy.storyteller.ai
+
+# Deprecated development hostnames :
+127.0.0.1    jungle.horse
+127.0.0.1    api.jungle.horse
+```
+
+### Nginx installation and administration
+
+Configure Nginx per the checked in Nginx configs (and instructions) in
+the directory `/_develop/localdev/nginx-http-config`.
+
+This should be straightforward, but if it isn't, contribute docs here.
+
+### [Fix] Python 3.6 on Apple M1 Mac
+
+Python3.6 isn't supported on Apple silicon, and it's not in homebrew. It can be installed with 
+[nix using Rosetta](https://stackoverflow.com/a/65980989):
+
+Download: https://nixos.org/download.html#nix-quick-install
+
+```
+nix run nixpkgs.python36 -c python
+```
+
+Install venv:
+
+```
+nix run nixpkgs.python36 -c python -m venv python
+```
+
+Install other packages on Mac that aren't used in venv:
+
+```
+python3 -m pip install --user requests gdown youtube_dl
+```

_develop/docs/local_development_setup.md

@@ -1,248 +0,0 @@
-dev setup
-=========
-
-[Back to main README](../../README.md)
-
-## Debugging and Fixing CUDA/PyTorch
-
-See this spreadsheet for previous battles with version incompatibility:
-https://docs.google.com/spreadsheets/d/1BEdLmwOzo3r83-iJn9sj6co1VT92t3lIeJ-lNH25bdQ/edit#gid=0
-
-Ubuntu might upgrade the driver by accident, and it might need reinstallation. Who knows.
-This stuff is a nightmare.
-
-## Database Setup
-
-### Mac Install
-
-```
-brew install mysql
-# If mysql -uroot fails, reboot the machine:
-sudo reboot now
-```
-
-### Linux Install
-
-Install the following libraries, and see the notes further below about MySQL on Ubuntu 20.04.
-
-```
-jq # for combining mysql codegen outputs
-libmysqlclient-dev
-mysql-server
-libsqlite3-dev
-pkgconf # if using openssl instead of rustls
-```
-
-### Migrations
-
-To manage the database and perform migrations, install the Rust tools diesel and sqlx.
-
-We'll be using diesel to manage the migrations, but sqlx within the app to actually perform queries.
-Diesel is an ORM, which is dumb, so we use sqlx as at-compile-time typesafe SQL.
-
-The CLI is necessary to perform codegen:
-
-```
-sudo apt-get install libmysqlclient-dev
-cargo install sqlx-cli --no-default-features --features rustls,mysql,sqlite
-```
-
-Diesel now supports configuring the migration directory via an environment variable,
-so we can install the currently published version:
-
-```
-cargo install diesel_cli \
-  --no-default-features \
-  --features mysql,sqlite
-```
-
-If the modern version fails, try installing a cherry-picked version with a 
-TOML-based migration directory configuration:
-
-```
-cargo install diesel_cli \
-  --git https://github.com/diesel-rs/diesel.git \
-  --rev a213fe232a122f35a812b0ce0269708a1845a4c9 \
-  --no-default-features \
-  --features mysql
-```
-
-#### Sqlx Error on Linux when Performing Schema Migrations
-
-You might get this error message during migration,
-
-```
-Encountered unknown type for Mysql: enum
-thread 'main' panicked at 'internal error: entered unreachable code: Mysql only supports a closed set of types.
-                         If you ever see this error message please open an issue at https://github.com/diesel-rs/diesel containing a dump of your schema definition.', diesel_cli/src/print_schema.rs:310:17
-note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
-```
-
-I haven't found the cause (it doesn't happen on Mac), but the migrations appear to work regardless of the error message.
-
-#### Linux database notes
-
-If MySql in local dev can't be connected to, reset the accounts:
-
-https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux
-
-#### MySql on Ubuntu 22.04
-
-Should work largely out of the box. We'll need a dev account:
-
-```
-# Connect to mysql:
-# sudo mysql -u root -p (password is "root")
-
-use mysql;
-CREATE USER 'storyteller'@'localhost' IDENTIFIED BY 'password';
-GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';
-```
-
-Then verify with `./dev_mysql_connect.sh`
-
-#### Fixing dev MySql on Ubuntu 20.04
-
-For some reason, the MySql default install on 20.04 gave me a bunch of trouble.
-
-In retrospect, I _think_ this is because 'root@localhost' requires sudo to access, but if this
-gives any trouble in the future, here's how I got around it (two full hours of distraction!)
-
-```
-sudo apt-get install mysql-server
-
-# But for some reason the default password doesn't work and diesel/sqlx can't connect?
-
-# Kill everything
-
-sudo apt purge mysql-server mysql-client mysql-common
-sudo apt autoremove
-sudo mv -iv /var/lib/mysql /var/tmp/mysql-backup
-sudo rm -rf /var/lib/mysql*
-
-sudo /usr/bin/mysql_secure_installation
-
-# still no work...
-
-sudo systemctl stop mysql.service
-
-# ugh, it wasn't chowned or created (both states observed in different installs)…
-sudo mkdir -p /var/run/mysqld
-sudo chown mysql:mysql /var/run/mysqld
-
-sudo mysqld_safe --skip-grant-tables --skip-networking &
-
-mysql -u root
-
-flush privileges;
-USE mysql;
-ALTER USER 'root'@'localhost' IDENTIFIED BY 'root';
-
-sudo killall -u mysql
-sudo systemctl restart mysql.service
-
-# now it works
-sudo mysql -u root -p
-
-# but now I have to use "sudo" !?!?
-
-use mysql;
-
-CREATE USER 'storyteller'@'localhost' IDENTIFIED BY 'password';
-GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';
-```
-
-### Install Diesel (migrations only)
-
-We use Diesel to manage migrations, but we don't use it in server code.
-Actual server code uses SQLx. To install the CLI tool, run the following:
-
-```
-cargo install diesel_cli
-```
-
-Mac [has issues](https://github.com/diesel-rs/diesel/issues/2605) and requires a few dependencies:
-
-```
-brew install libpq
-```
-
-### Database migrations
-
-To reset the entire database (drop, migrate), run:
-
-```
-diesel database reset
-```
-
-To migrate at the current step and beyond:
-
-```
-diesel migration run
-```
-
-To undo migrations at the current step:
-
-```
-diesel migration redo
-```
-
-### Server Query Codegen
-
-We use SQLx instead of Diesel in the production server. It's typesafe
-SQL instead of an ORM like Diesel.
-
-SQLx connects to a database to derive type information, but obviously
-cannot do this for builds in CI. In order to cache the types, we build
-and check in a cache file (necessary for builds):
-
-```
-SQLX_OFFLINE=true cargo sqlx prepare
-```
-
-Now that we have multiple binaries, it's required to include all the queries in the main
-binary so we can generate the cached queries as a single target. That's then executed
-with:
-
-```
-SQLX_OFFLINE=true cargo sqlx prepare -- --bin storyteller-web
-```
-
-### Setting Up a Local Nginx Proxy
-
-Set up a local nginx to proxy to the frontend and backend so cookie issues aren't annoying
-
-Configure Nginx per the checked in Nginx configs (and instructions) in `localdev/nginx-http-config`.
-
-```
-
-And in /etc/hosts,
-
-```
-127.0.0.1  jungle.horse
-127.0.0.1  api.jungle.horse
-
-```
-
-### Python 3.6 on Apple M1 Mac
-
-Python3.6 isn't supported on Apple silicon, and it's not in homebrew. It can be installed with 
-[nix using Rosetta](https://stackoverflow.com/a/65980989):
-
-Download: https://nixos.org/download.html#nix-quick-install
-
-```
-nix run nixpkgs.python36 -c python
-```
-
-Install venv:
-
-```
-nix run nixpkgs.python36 -c python -m venv python
-```
-
-Install other packages on Mac that aren't used in venv:
-
-```
-python3 -m pip install --user requests gdown youtube_dl
-```

_develop/docs/old_readme.md

@@ -1,77 +0,0 @@
-old readme
-==========
-
-Contents of the main README.md, which will always be woefully out of date. Docs will migrate into this folder on a 
-per-subject basis. App-specific docs will live under each app.
-
-Application overview
---------------------
-
-* [FakeYou.com](https://fakeyou.com) is powered by:
-    * HTTP APIs:
-        * `storyteller-web` - HTTP + database monolith for FakeYou
-    * Asynchronous Jobs:
-        * `tts-download-job` - Async download of models
-        * `tts-inference-job` - Async TTS inference
-        * `w2l-download-job` - Async download of videos and images
-        * `w2l-inference-job` - Async W2L inference
-
-* [StorytellerStream](https://storyteller.stream) is powered by:
-    * HTTP APIs:
-        * `storyteller-web` - HTTP + database monolith for FakeYou
-    * Websocket events:
-        * `websocket-gateway` - Websocket to pull down Twitch events.
-    * Twitch API integration:
-        * `twitch-pubsub-subscriber` - Dynamically subscribes to Twitch PubSub for "active" users
-
-Schema and API notes
---------------------
-
-* [Redis Schema Notes README](_docs/redis_schema.md)
-
-Local development
------------------
-
-[Local development setup README](_docs/local_development_setup.md)
-
-Production and Deployment
--------------------------
-
-[Production builds and deployment README](_docs/production_builds_and_deployment.md)
-
-Development notes
------------------
-
-[Development notes README](_docs/development_notes.md)
-
-Dump a MySQL report
--------------------
-
-```
-mysql -u storyteller \
-  -pPasswordHere \
-  -h IP_HERE \
-  -P 3306 \
-  -D storyteller \
-  -B \
-  -e "select * from voice_clone_requests;" | sed "s/'/\'/;s/\t/\",\"/g;s/^/\"/;s/$/\"/;s/\n//g" > voice_clone_report.csv
-```
-
-TODO
-----
-
-* Revise this README to be more useful.
-
-Notes / TODOs:
-
-* Examples for good Actix+Sqlx Tests:
-  https://stackoverflow.com/questions/65370752/how-do-i-create-an-actix-web-server-that-accepts-both-sqlx-database-pools-and-tr
-
-* Actix/sqlx runtime compat:
-  https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734
-
-* Redis caching of more endpoints
-
-* Jobs for analytics queries (#uses per model, etc)
-* Jobs for ingesting Discord, Twitter, Reddit RPAN chat, Twitch chat (not PubSub), etc.
-

_develop/docs/production/production_database.md

@@ -0,0 +1,17 @@
+Production Database
+===================
+
+Secrets can be subbed in for `.env` and the standard diesel migration tooling will work.
+
+Dump a MySQL report
+-------------------
+
+```bash
+mysql -u storyteller \
+  -pPasswordHere \
+  -h IP_HERE \
+  -P 3306 \
+  -D storyteller \
+  -B \
+  -e "select * from voice_clone_requests;" | sed "s/'/\'/;s/\t/\",\"/g;s/^/\"/;s/$/\"/;s/\n//g" > voice_clone_report.csv
+```

_develop/docs/schema/database_schema.md

@@ -1,6 +1,8 @@
 database schema
 ================
 
+(This is woefully incomplete and unmaintained.)
+
 Twitch Model
 ------------
 

_sql/actions/generic_download_jobs.sql

@@ -0,0 +1,48 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- 1) Find uploads that failed (recently)
+
+select
+    id,
+    token,
+    download_type,
+    title,
+    download_url,
+    status,
+    attempt_count
+from generic_download_jobs
+where status IN ('started', 'attempt_failed', 'complete_failure', 'dead')
+and download_type NOT IN ('vits', 'hifigan')
+and created_at > NOW() - INTERVAL 15 DAY;
+
+
+-- 2) Then restart them
+-- NB: Double inner query is to get away from query optimizer:
+--  "You can't specify target table 'generic_download_jobs' for update in FROM clause"
+
+update generic_download_jobs
+set
+    status='pending',
+    attempt_count=0
+where token IN (
+    select token from (
+        select token
+        from generic_download_jobs
+        where status IN ('started', 'attempt_failed', 'complete_failure', 'dead')
+        and download_type NOT IN ('vits', 'hifigan')
+        and created_at > NOW() - INTERVAL 15 DAY
+    ) as x
+);
+
+
+-- Kill bad pending downloads
+-- eg. awful voices, like Hitler
+
+update generic_download_jobs
+set
+    status='dead',
+    download_url = CONCAT('NOPE', download_url)
+where token = 'jdown_ssape40dxbfgvhtsmedt34418x'
+limit 1;

_sql/actions/generic_inference_jobs.sql

@@ -7,3 +7,7 @@ update generic_inference_jobs set status = 'dead' where maybe_routing_tag IS NOT
 
 -- Get pending jobs
 select count(*) from generic_inference_jobs where status = 'pending';
+
+update generic_inference_jobs
+set status = 'dead'
+where status IN ('pending', 'started', 'attempt_failed');

_sql/actions/voice_conversion_models.sql

@@ -46,10 +46,267 @@ where token IN (
     'vcm_xsx5yx585c7x',
     'vcm_dpahzjjpkf18',
     'vcm_y09dqe6zgwyk',
-    'vcm_xrp96dwgppv2'
+    'vcm_xrp96dwgppv2',
+    'vcm_84awjbjxts6n',
+    'vcm_b2x2sn8kf28j',
+    'vcm_41nw6xgr5x3v',
+    'vcm_a6x83tk0ttf2',
+    'vcm_dgjgqxq5w97j',
+    'vcm_59k5j452x7xy',
+    'vcm_09p6vqnk7n05',
+    'vcm_attanwh383hg',
+    'vcm_t8d4f6w9pa8t',
+    'vcm_s8s4q4c65xsy',
+    'vcm_6e2av8pmkyz5',
+    'vcm_82x5f330z4wf',
+    'vcm_z6bax4kvjdx6',
+    'vcm_v8hf35cyq3ft',
+    'vcm_1h9b4v6nxmf5',
+    'vcm_781bzn81qf6k',
+    'vcm_w0xjjsexktby',
+    'vcm_awsq3q4c711w',
+    'vcm_kttxt7wgb6s6',
+    'vcm_ets26z5fpt8d',
+    'vcm_wjrw59wmwcrr',
+    'vcm_bebgka17n0sr',
+    'vcm_6mj0x3qd7yfd',
+    'vcm_b71ke4fm61nx',
+    'vcm_srz3gas5ra4d',
+    'vcm_k64jj2t2j05b',
+    'vcm_140rd035642m',
+    'vcm_5y93wgr90s91',
+    'vcm_jwp6p08byjn9',
+    'vcm_ets26z5fpt8d',
+    'vcm_7yfwzsyvrpq4',
+    'vcm_f1vq0hp0tes2',
+    'vcm_9f05kg36q4fg',
+    'vcm_1xs9c2sgykgy',
+    'vcm_hnd9z2fxq661',
+    'vcm_jxd74xhcszry',
+    'vcm_tvk0g6dwg3yv',
+    'vcm_vrsdv4ahterx',
+    'vcm_e2debszbm8vt',
+    'vcm_rh98fw9a9an1',
+    'vcm_2bvd1gmysaxf',
+    'vcm_2cqq1j33y2hr',
+    'vcm_ysj7kwb96kcp',
+    'vcm_ef0eaneyr72p',
+    'vcm_t59d7vdxtqs4',
+    'vcm_90hrepevshtg',
+    'vcm_4st9ex13vnbh',
+    'vcm_30fne697qhab',
+    'vcm_qy7evy03p6tg',
+    'vcm_775eze19tv0n',
+    'vcm_4bzkfvaexbv3',
+    'vcm_08v2tdga8v93',
+    'vcm_whxeyxzw12q6',
+    'vcm_0rj5b6npxfkh',
+    'vcm_hv22emyfkex4',
+    'vcm_a1fmtgthezs0',
+    'vcm_z9fpz1p5bpr2',
+    'vcm_2y8ck2dbppkq',
+    'vcm_0c9h3mzc4kre',
+    'vcm_m0z3tt4gytm9'
+);
+
+-- Remove duplicate models (2)
+update voice_conversion_models
+set
+    mod_deleted_at = NOW(),
+    maybe_mod_comments = 'duplicate model'
+where token IN (
+    'vcm_v790w43dv3ce',
+    'vcm_xmzm7eextc3n',
+    'vcm_r3b91dr0eta5',
+    'vcm_p1rpmfmckd5c',
+    'vcm_cpajdgenjfw7',
+    'vcm_qchjs52fsr9y',
+    'vcm_pkkjy8c6cpt3',
+    'vcm_f8ra6j8d6wgv',
+    'vcm_mhsz9bjn19tw',
+    'vcm_btd5zab2jdqv',
+    'vcm_q5evptaaqne3',
+    'vcm_wqtqs4gj2zmj',
+    'vcm_14rv35fct2ay',
+    'vcm_418j0h41qy8a',
+    'vcm_3m5m6bpm0rt2',
+    'vcm_gdw4krmaxpza',
+    'vcm_b3xhdb8zymvf',
+    'vcm_aak9m6nvj3am',
+    'vcm_mcw5kn21bbjh',
+    'vcm_fydr6ts21mtf',
+    'vcm_zpw560jzyqpw',
+    'vcm_w24wcctnxbxf',
+    'vcm_ya5tpzdcvvcz',
+    'vcm_1ctmgrjtn1j7',
+    'vcm_rk8fycc0reb2',
+    'vcm_e2c8942vky34',
+    'vcm_wevg8cf3hcw8',
+    'vcm_6hzbyg453xb9',
+    'vcm_xqfffshe2g0q',
+    'vcm_32j1zmcvn61n'
+);
+
+-- Remove duplicate models (3)
+update voice_conversion_models
+set
+    mod_deleted_at = NOW(),
+    maybe_mod_comments = 'duplicate model'
+where token IN (
+    'vcm_46sbq0r9hfjk',
+    'vcm_k8nseqy8jyte',
+    'vcm_gs8qam6npe46',
+    'vcm_djh15g49cxm8',
+    'vcm_d7zygjnp35my',
+    'vcm_334yqeq11rpm',
+    'vcm_6m3b6121318k',
+    'vcm_a419w7gw6jrt',
+    'vcm_wtyszwjwq0ts',
+    'vcm_0q9t2azhqvf1',
+    'vcm_fx2k40ffzjqg',
+    'vcm_wvjqmzd8y4mt',
+    'vcm_b2635v3p50je',
+    'vcm_zfyr07z2d0rb',
+    'vcm_yn2sn5j2jwed',
+    'vcm_z596ja3yvyy9',
+    'vcm_a1p854bn1sjs',
+    'vcm_fv24vmr93805',
+    'vcm_0y7vsc2e9s3r',
+    'vcm_5aqzq0d246rm',
+    'vcm_trvge62x7je9',
+    'vcm_sghga68kwggk',
+    'vcm_raxcc33pj608',
+    'vcm_a66b3va0sj64',
+    'vcm_bd1rjawqypw6',
+    'vcm_evzff14n721p',
+    'vcm_jnrr4ht9f42f'
+);
+
+-- Remove duplicate models (4)
+update voice_conversion_models
+set
+    mod_deleted_at = NOW(),
+    maybe_mod_comments = 'duplicate model'
+where token IN (
+    'vcm_ds2pz63sagan',
+    'vcm_wkczgvq2fjqq',
+    'vcm_z8137w1rddse',
+    'vcm_gknzxybf4erw',
+    'vcm_z0m1qtmawccn',
+    'vcm_yek6pmgvbb1s',
+    'vcm_qqhnjwsjrvnx',
+    'vcm_8fetm7qz94cp',
+    'vcm_kz1zm1d269j0',
+    'vcm_rfnkc8g5gbbv',
+    'vcm_4ncqxetfe7dm',
+    'vcm_qkpfem2a153n',
+    'vcm_z5jkd9r4cawf',
+    'vcm_vgh6ja6zzj0p',
+    'vcm_bd473tdrrzhk',
+    'vcm_2k7tmx3ndd3y',
+    'vcm_weyxz516dd2m',
+    'vcm_1j4kpjvgsj6k',
+    'vcm_211g17ea44fw',
+    'vcm_7zkbj6be0yr5',
+    'vcm_nnaaxeqy4xj6',
+    'vcm_wp7sgtz6mhc4',
+    'vcm_h5w7fzhxcpxm',
+    'vcm_7d5rxpqkjy07',
+    'vcm_9vhjvrptygfz',
+    'vcm_4qbbawahnkvb',
+    'vcm_am870m8162at',
+    'vcm_cftdb5227qt5',
+    'vcm_m7vv1mcxfxvm',
+    'vcm_qn8vsvvh1rqc',
+    'vcm_erhvqjvk1dsk',
+    'vcm_4zs8tdrw971e',
+    'vcm_4zs8tdrw971e',
+    'vcm_z8htjaejr1wx',
+    'vcm_99wdsazefjwr',
+    'vcm_k1at1sbepvqv',
+    'vcm_xcp34f5d7ygs',
+    'vcm_8v8srq9sx7cq',
+    'vcm_f2yf7bxmk8h5',
+    'vcm_gvtrjte4as1s',
+    'vcm_2s0y50avy43b',
+    'vcm_r638ygfxkpmq',
+    'vcm_1ttm6pxcpqna',
+    'vcm_0ytfkxh6pm4f',
+    'vcm_zderdrfmgq6c',
+    'vcm_jvdpr6mtddb8',
+    'vcm_9tvcv961g09x',
+    'vcm_aynpmxbhpy7n',
+    'vcm_2gzsjb5bdsdq',
+    'vcm_a0vep64tz56n',
+    'vcm_zt4hgc9pyv11',
+    'vcm_szfxm0crd8y8',
+    'vcm_3q89kba3cy5z',
+    'vcm_ywpwnhk4pv4x',
+    'vcm_4he8jjmengb3',
+    'vcm_jpdfswn0empy',
+    'vcm_2nztkt3rgm1q',
+    'vcm_q5gyxf5a11zf',
+    'vcm_sa7k9ycne147',
+    'vcm_p6gxtj9qefsg',
+    'vcm_j2fvzdyachtw',
+    'vcm_ecbv0rhc6br3',
+    'vcm_jrgwspz2wt7p',
+    'vcm_f5b4j3rt180n',
+    'vcm_pzajxdfcjfpa',
+    'vcm_rq4f2cbbm6j1',
+    'vcm_rdmfpbnab47w',
+    'vcm_03dtk4xtggm6',
+    'vcm_zx7mdkh9s9qb',
+    'vcm_e0megdtf71x6'
+);
+
+-- Remove older models replaced by new versions
+update voice_conversion_models
+set
+    mod_deleted_at = NOW(),
+    maybe_mod_comments = 'replaced with new version'
+where token IN (
+    'vcm_gdg3najz6j3g'
+);
+
+-- Remove takedown request models
+update voice_conversion_models
+set
+    mod_deleted_at = NOW(),
+    maybe_mod_comments = 'takedown request'
+where token IN (
+    'vcm_2af74np3jh6q',
+    'vcm_8kk7355dc184',
+    'vcm_nsh89pcx4569',
+    'vcm_q7ewn3yym4gy',
+    'vcm_cwngb21xsf66',
+    'vcm_jmzjrky179cm',
+    'vcm_hj6a1nt1yvh9',
+    'vcm_6wwx1e3cd48s',
+    'vcm_pc76gc1xh7jg',
+    'vcm_gb0p1wmtk9wp',
+    'vcm_pbs955dvaeq6',
+    'vcm_x2wyzr6h6539',
+    'vcm_py7qwz310qff',
+    'vcm_k01bqvh0dseq',
+    'vcm_5m4dsn2fwa0q',
+    'vcm_hn9m98g1am3z',
+    'vcm_f9168tyyn6ex',
+    'vcm_03y9w4h99r1h',
+    'vcm_yr9f1s0xc7pc',
+    'vcm_45767r26a9bb',
+    'vcm_rq3cvp3x3hnh',
+    'vcm_ng1nz8d67z4v',
+    'vcm_f5arabzrk4en',
+    'vcm_d379600cr8yx',
+    'vcm_p7m9jw1krrkr',
+    'vcm_x7v3m9mcvten',
+    'vcm_6r057m7594bv',
+    'vcm_wkm9e42d3spm'
 );
 
 
 
 
 
+

_sql/actions/w2l_inference_jobs.sql

@@ -17,3 +17,9 @@ update w2l_template_upload_jobs
 set status='dead'
 where status='pending'
   and download_url LIKE '%tiktok%';
+
+
+-- Kill all pending jobs
+update w2l_template_upload_jobs
+set status='dead'
+where status IN ('pending', 'started', 'attempt_failed');

_sql/analytics/analytics_stripe_sigma/basic_user_queries.sql

@@ -28,3 +28,9 @@ where id IN (
     or subscriptions.status = 'past_due'
 )
 group by address_country
+
+
+-- Histogram
+select address_country, count(*)
+from customers
+group by address_country

_sql/analytics/analytics_stripe_sigma/subscriptions.sql

@@ -0,0 +1,90 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- NB: These are for use in Stripe Sigma, not MySQL
+-- NB: You must use single quotes instead of double quotes when comparing string values.
+
+-- Status values: past_due, incomplete, active, canceled, incomplete_expired (perhaps not exhaustive)
+select distinct status
+from subscriptions
+
+select count(*)
+from subscriptions
+where status IN ('active');
+
+-- Metadata is in the 'subscriptions_metadata' table, and we store 'username', 'email', and 'user_token'
+select
+    m.value,
+    m.key,
+    s.id,
+    s.created,
+    s.cancel_at,
+    s.canceled_at
+from subscriptions as s
+join subscriptions_metadata as m
+on m.subscription_id = s.id
+where m.key = 'username'
+
+
+-- Full mapping of Stripe subscribers to FakeYou users
+select
+    s.id as subscription_id,
+    user_tokens.user_token,
+    usernames.username,
+    emails.email,
+    s.status as subscription_status,
+    s.created,
+    s.cancel_at,
+    s.canceled_at
+from subscriptions as s
+join (
+    select
+        m.subscription_id,
+        m.value as username
+    from subscriptions_metadata as m
+    where m.key = 'username'
+) as usernames
+on s.id = usernames.subscription_id
+join (
+    select m.subscription_id,
+           m.value as email
+    from subscriptions_metadata as m
+    where m.key = 'email'
+) as emails
+on s.id = emails.subscription_id
+join (
+    select m.subscription_id,
+           m.value as user_token
+    from subscriptions_metadata as m
+    where m.key = 'user_token'
+) as user_tokens
+on s.id = user_tokens.subscription_id
+order by s.id asc
+
+-- Without emails
+select
+    s.id as subscription_id,
+    user_tokens.user_token,
+    usernames.username,
+    s.status as subscription_status,
+    s.created,
+    s.cancel_at,
+    s.canceled_at
+from subscriptions as s
+join (
+    select
+        m.subscription_id,
+        m.value as username
+    from subscriptions_metadata as m
+    where m.key = 'username'
+) as usernames
+on s.id = usernames.subscription_id
+join (
+    select m.subscription_id,
+           m.value as user_token
+    from subscriptions_metadata as m
+    where m.key = 'user_token'
+) as user_tokens
+on s.id = user_tokens.subscription_id
+order by s.id asc

_sql/analytics/analytics_stripe_sigma/user_cohorts.sql

@@ -0,0 +1,8 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- NB: These are for use in Stripe Sigma, not MySQL
+-- NB: You must use single quotes instead of double quotes when comparing string values.
+
+

_sql/analytics/generic_inference/generic_inference_jobs.sql

@@ -33,6 +33,7 @@ from (
              jobs.assigned_worker,
              jobs.assigned_cluster,
              jobs.maybe_input_source_token,
+             jobs.maybe_model_type,
              jobs.maybe_model_token,
              m.media_source,
              m.maybe_original_mime_type as mime_type,
@@ -50,7 +51,7 @@ from (
              jobs.status != 'pending'
         AND jobs.created_at > NOW() - INTERVAL 20 MINUTE
          order by id desc
-             limit 5000
+             limit 200
      ) as t
 order by execution_mins desc;
 
@@ -78,3 +79,118 @@ select
 from generic_inference_jobs
 where maybe_routing_tag IS NOT NULL
     limit 10;
+
+
+
+-- Debug that we're setting the correct metadata on jobs
+-- For some reasons storyteller-web is enqueuing the wrong type!
+select
+    jobs.id,
+    jobs.token,
+    jobs.maybe_model_type as jobs_model_type,
+    jobs.maybe_model_token as jobs_model_token,
+    models.token as model_token,
+    models.model_type as model_type
+from generic_inference_jobs as jobs
+left outer join voice_conversion_models as models
+on jobs.maybe_model_token = models.token
+order by jobs.id
+desc
+limit 100;
+
+-- Generic inference jobs + legacy TTS (this is a production endpoint)
+-- UPDATED QUERY: Queues as named rows
+        SELECT
+            maybe_model_type as queue_type,
+            count(*) as pending_job_count,
+            NOW() as present_time
+        FROM (
+            SELECT
+                token,
+                maybe_model_type
+            FROM generic_inference_jobs
+            WHERE status IN ("pending", "attempt_failed")
+            UNION
+            SELECT
+                token,
+                maybe_model_type
+            FROM generic_inference_jobs
+            WHERE status IN ("started")
+            AND created_at > (CURDATE() - INTERVAL 15 MINUTE)
+        ) as generic_inner
+        GROUP BY queue_type
+UNION
+        SELECT
+            "legacy_tts" as queue_type,
+            count(*) as pending_job_count,
+            NOW() as present_time
+        FROM
+        (
+             SELECT token
+             FROM tts_inference_jobs
+             WHERE status IN ("pending", "attempt_failed")
+             UNION
+             SELECT token
+             FROM tts_inference_jobs
+             WHERE status IN ("started")
+             AND created_at > (CURDATE() - INTERVAL 15 MINUTE)
+        ) as legacy_inner
+        GROUP BY queue_type;
+
+-- Generic inference jobs + legacy TTS (this was a production endpoint)
+-- OLD QUERY: Queues as columns
+SELECT
+(
+    SELECT
+        COUNT(distinct token) as generic_job_count
+    FROM
+        (
+            SELECT token
+            FROM generic_inference_jobs
+            WHERE status = "started"
+              AND created_at > (CURDATE() - INTERVAL 5 MINUTE)
+            UNION
+            SELECT token
+            FROM generic_inference_jobs
+            WHERE status IN ("pending", "attempt_failed")
+        ) as g
+) as generic_job_count,
+(
+    SELECT
+        COUNT(distinct token) as legacy_tts_job_count
+    FROM
+        (
+            SELECT token
+            FROM tts_inference_jobs
+            WHERE status = "started"
+              AND created_at > (CURDATE() - INTERVAL 5 MINUTE)
+            UNION
+            SELECT token
+            FROM tts_inference_jobs
+            WHERE status IN ("pending", "attempt_failed")
+        ) as t
+) as legacy_tts_job_count,
+NOW() as present_time;
+
+-- Histogram of inference
+select
+    date(created_at) as created_date,
+    count(*) as job_count
+from generic_inference_jobs
+group by created_date
+
+-- Histogram of inference (within range)
+select
+    date(created_at) as created_date,
+    count(*) as job_count
+from generic_inference_jobs
+where created_at > (CURDATE() - INTERVAL 5 DAY)
+group by created_date
+
+-- Histogram of inference (within range)
+select
+    date(created_at) as created_date,
+    count(*) as job_count
+from generic_inference_jobs
+where date(created_at) >= "2023-01-01"
+group by created_date

_sql/analytics/tts/tts_inference_jobs.sql

@@ -21,6 +21,23 @@ group by creator_ip_address
 order by attempts desc
 limit 50;
 
+-- Top (IP addresses, usernames) making requests
+select distinct j.creator_ip_address, u.username, count(*) as attempts
+from tts_inference_jobs AS j
+left outer join users AS u
+on u.token = j.maybe_creator_user_token
+where j.status = 'pending'
+group by j.creator_ip_address, u.username
+order by attempts desc
+    limit 50;
+
+-- Find a job by an IP
+select *
+from tts_inference_jobs
+where status = 'pending'
+and creator_ip_address = '186.188.28.73'
+limit 1;
+
 -- Top voices in requests
 select distinct model_token, count(*) as attempts
 from tts_inference_jobs

_sql/analytics/tts/tts_models.sql

@@ -307,6 +307,21 @@ from (
 order by m.created_at desc
     limit 100;
 
+-- Histogram of model contributions
+select
+    date(created_at) as created_date,
+    count(*) as model_count
+from tts_models
+group by created_date
+
+-- Histogram of model contributions (within range)
+select
+    date(created_at) as created_date,
+    count(*) as model_count
+from tts_models
+where created_at > (CURDATE() - INTERVAL 5 DAY)
+group by created_date
+
 --
 -- Models deleted recently (voice actor take down)
 --

_sql/analytics/tts/tts_results.sql

@@ -31,6 +31,29 @@ from (
 ) as sample
 where sample.is_generated_on_prem IS TRUE;
 
+-- Histogram of days of content per day
+select
+    date(created_at) as created_date,
+    sum(duration_millis) / 1000 / 60 / 60 / 24 as days
+from tts_results
+group by created_date
+
+-- Histogram of days of content per day (range)
+select
+    date(created_at) as created_date,
+    sum(duration_millis) / 1000 / 60 / 60 / 24 as days
+from tts_results
+where date(created_at) >= "2023-01-01"
+group by created_date
+
+-- Histogram of days of content per day (range)
+-- Maybe non-tablescan?
+select
+    date(created_at) as created_date,
+    sum(duration_millis) / 1000 / 60 / 60 / 24 as days
+from tts_results
+where created_at > (CURDATE() - INTERVAL 30 DAY)
+group by created_date
 
 -- Find TTS results for a single model
 select count(*) from tts_results
@@ -96,7 +119,7 @@ WHERE
 
 --- Search for 101soundboards.com using us
 SELECT
-    r.maybe_creator_user_token, r.creator_ip_address
+    r.maybe_creator_user_token, r.creator_ip_address, r.raw_inference_text
 FROM
     tts_results as r
         LEFT OUTER JOIN

_sql/analytics/users/subscribers/loyal_user_cohorts_mom.sql

@@ -0,0 +1,56 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Example query for date ranges:
+select count(*)
+from users
+where created_at BETWEEN '2023-07-01 00:00:00' AND '2023-07-30 00:00:00';
+
+--
+
+-- Month by month subscriber usages.
+SELECT
+    username,
+    count(*) as total_use_count
+FROM (
+         SELECT
+             u.username as username,
+             j.token as token
+         FROM users AS u
+                  JOIN tts_inference_jobs AS j
+                       ON u.token = j.maybe_creator_user_token
+         WHERE u.token IN (
+             SELECT DISTINCT user_token
+             FROM user_subscriptions
+         )
+           AND j.created_at BETWEEN '2022-12-01 00:00:00' AND '2023-01-01 00:00:00'
+         UNION
+         SELECT
+             u.username as username,
+             j.token as token
+         FROM users AS u
+             JOIN generic_inference_jobs AS j
+         ON u.token = j.maybe_creator_user_token
+         WHERE u.token IN (
+             SELECT DISTINCT user_token
+             FROM user_subscriptions
+             )
+           AND j.created_at BETWEEN '2022-12-01 00:00:00' AND '2023-01-01 00:00:00'
+         UNION
+         SELECT
+             u.username as username,
+             j.token as token
+         FROM users AS u
+             JOIN w2l_inference_jobs AS j
+         ON u.token = j.maybe_creator_user_token
+         WHERE u.token IN (
+             SELECT DISTINCT user_token
+             FROM user_subscriptions
+             )
+           AND j.created_at BETWEEN '2022-12-01 00:00:00' AND '2023-01-01 00:00:00'
+     ) as t
+GROUP BY username
+ORDER BY total_use_count desc;
+
+

_sql/analytics/users/subscribers/paying_user_interests.sql

@@ -37,7 +37,7 @@ FROM (
         x.use_count > 10;
 
 
--- The first voice a paying customer uses
+-- The first TTS voice a paying customer uses
 -- TODO: List *all* of the voices a paying customer uses before they subscribe, not just the first.
 -- select first row of partially aggregate query: https://stackoverflow.com/a/73157541
 SELECT *
@@ -78,6 +78,48 @@ FROM (
      ) AS z
 ORDER BY use_count desc
 
+
+-- The first VC voice a paying customer uses
+-- TODO: List *all* of the voices a paying customer uses before they subscribe, not just the first.
+-- select first row of partially aggregate query: https://stackoverflow.com/a/73157541
+SELECT *
+FROM (
+         SELECT
+             m.token,
+             m.title,
+             count(*) as use_count,
+             m.created_at,
+             m.user_deleted_at,
+             m.mod_deleted_at
+         FROM voice_conversion_models AS m
+                  JOIN
+              (
+                  SELECT
+                      model_token
+                  FROM (
+                           SELECT
+                               maybe_model_token as model_token,
+                               maybe_creator_user_token,
+                               row_number() over(partition by maybe_creator_user_token order by id asc) as rownum
+                           FROM
+                               generic_inference_jobs
+                           WHERE maybe_model_token IS NOT NULL
+                           AND maybe_creator_user_token IN
+                                 (
+                                     select distinct user_token
+                                     from user_subscriptions
+                                     where created_at > (CURDATE() - INTERVAL 2 DAY)
+                                 )
+                       ) AS x
+                  WHERE rownum = 1
+                  ORDER BY model_token desc
+              ) AS y
+              ON m.token = y.model_token
+         GROUP BY m.token
+     ) AS z
+ORDER BY use_count desc
+
+
 -- The last voice a paying customer uses
 -- select first row of partially aggregate query: https://stackoverflow.com/a/73157541
 SELECT *

_sql/analytics/users/top_users_by_usage.sql

@@ -0,0 +1,53 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+SELECT
+    u.username as username,
+    j.token as token
+FROM users AS u
+JOIN tts_inference_jobs AS j
+ON u.token = j.maybe_creator_user_token
+  AND j.created_at > NOW() - INTERVAL 31 DAY
+
+
+SELECT
+    t.maybe_creator_user_token as user_token,
+    u.username,
+    count(*) as usage_count,
+    u.created_at
+FROM tts_results as t
+JOIN users as u
+ON u.token = t.maybe_creator_user_token
+WHERE t.maybe_creator_user_token IS NOT NULL
+GROUP BY t.maybe_creator_user_token
+ORDER BY usage_count DESC
+LIMIT 50000;
+
+
+SELECT
+    v.maybe_creator_user_token as user_token,
+    u.username,
+    count(*) as usage_count
+FROM voice_conversion_results as v
+JOIN users as u
+ON u.token = v.maybe_creator_user_token
+WHERE v.maybe_creator_user_token IS NOT NULL
+GROUP BY v.maybe_creator_user_token
+ORDER BY usage_count DESC
+LIMIT 5000;
+
+
+SELECT
+    w.maybe_creator_user_token as user_token,
+    u.username,
+    count(*) as usage_count
+FROM w2l_results as w
+         JOIN users as u
+              ON u.token = w.maybe_creator_user_token
+WHERE w.maybe_creator_user_token IS NOT NULL
+GROUP BY w.maybe_creator_user_token
+ORDER BY usage_count DESC
+LIMIT 5000;
+
+

_sql/analytics/users/user_registrations.sql

@@ -0,0 +1,19 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- All users
+select count(*) from users;
+
+-- Users registered in the last year
+select count(*)
+from users
+where created_at > ( CURDATE() - INTERVAL 365 DAY )
+
+-- Histogram of user signups
+select
+    date(created_at) as created_date,
+    count(*) as registered
+from users
+where date(created_at) >= "2023-01-01"
+group by created_date

_sql/analytics/voice_conversion/voice_conversion_models.sql

@@ -34,4 +34,137 @@ left join users AS u on
     u.token = vc.creator_user_token
 where
     vc.user_deleted_at IS NULL
-    and vc.mod_deleted_at IS NULL;
+    and vc.mod_deleted_at IS NULL
+order by vc.title asc;
+
+-- Inventory duplicate non-deleted models
+select
+    vc.id,
+    vc.token,
+    vc.model_type,
+    vc.title,
+    u.username,
+    vc.original_download_url,
+    vc.created_at,
+    vc.updated_at
+from voice_conversion_models as vc
+         left join users AS u on
+            u.token = vc.creator_user_token
+where vc.original_download_url IN (
+    select original_download_url from (
+      select original_download_url,
+             count(*) as duplicate_count
+      from voice_conversion_models
+      where user_deleted_at IS NULL
+        and mod_deleted_at IS NULL
+      group by original_download_url
+      order by original_download_url desc
+    ) as duplicates
+    where duplicate_count > 1
+)
+and vc.user_deleted_at IS NULL
+and vc.mod_deleted_at IS NULL
+order by
+    vc.original_download_url asc,
+    id asc;
+
+-- order by vc.original_download_url ASC;
+
+--
+-- Top 100 voice conversion models by use count
+-- (jobs table, not results table)
+--
+select
+    m.token,
+    m.model_type,
+    m.title,
+    u.username,
+    r.use_count,
+    m.created_at,
+    m.user_deleted_at,
+    m.mod_deleted_at
+from (
+         select maybe_model_token, count(*) as use_count
+         from generic_inference_jobs
+         where maybe_model_token IS NOT NULL
+         group by maybe_model_token
+     ) as r
+         join voice_conversion_models as m
+              on m.token = r.maybe_model_token
+         join users as u
+              on u.token = m.creator_user_token
+order by r.use_count desc
+    limit 100;
+
+
+--
+-- Top 100 voice conversion models by use count (NON DELETED)
+-- (jobs table, not results table)
+--
+select
+    m.token,
+    m.model_type,
+    m.title,
+    u.username,
+    r.use_count,
+    m.created_at
+from (
+     select maybe_model_token, count(*) as use_count
+     from generic_inference_jobs
+     where maybe_model_token IS NOT NULL
+     group by maybe_model_token
+) as r
+join voice_conversion_models as m
+    on m.token = r.maybe_model_token
+join users as u
+    on u.token = m.creator_user_token
+where m.user_deleted_at IS NULL
+and m.mod_deleted_at IS NULL
+order by r.use_count desc
+limit 100;
+
+
+--
+-- Top 100 voice conversion models by use count (NON DELETED, LAST 7 DAYS)
+-- (jobs table, not results table)
+--
+select
+    m.token,
+    m.model_type,
+    m.title,
+    u.username,
+    r.use_count,
+    m.created_at
+from (
+    select maybe_model_token, count(*) as use_count
+    from generic_inference_jobs
+    where maybe_model_token IS NOT NULL
+    and created_at > ( CURDATE() - INTERVAL 7 day)
+    group by maybe_model_token
+) as r
+join voice_conversion_models as m
+    on m.token = r.maybe_model_token
+join users as u
+    on u.token = m.creator_user_token
+where m.user_deleted_at IS NULL
+  and m.mod_deleted_at IS NULL
+order by r.use_count desc
+    limit 100;
+
+
+
+-- Histogram of model contributions
+select
+    date(created_at) as created_date,
+    count(*) as model_count
+from voice_conversion_models
+group by created_date
+
+-- Histogram of model contributions (within range)
+select
+    date(created_at) as created_date,
+    count(*) as model_count
+from voice_conversion_models
+where created_at > (CURDATE() - INTERVAL 5 DAY)
+group by created_date
+

_sql/analytics/voice_conversion/voice_conversion_results.sql

@@ -0,0 +1,11 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Histogram of days of content per day
+select
+    date(created_at) as created_date,
+    sum(duration_millis) / 1000 / 60 / 60 / 24 as days
+from voice_conversion_results
+group by created_date
+

_sql/analytics/w2l/w2l_inference_jobs.sql

@@ -5,3 +5,26 @@
 -- Pending job count
 select count(*) from w2l_inference_jobs where status='pending';
 
+-- Histogram of inference
+select
+    date(created_at) as created_date,
+    count(*) as job_count
+from w2l_inference_jobs
+group by created_date
+
+-- Histogram of inference (within range)
+select
+    date(created_at) as created_date,
+    count(*) as job_count
+from w2l_inference_jobs
+where date(created_at) >= "2023-01-01"
+group by created_date
+
+-- Histogram of inference (within range)
+select
+    date(created_at) as created_date,
+    count(*) as job_count
+from w2l_inference_jobs
+where created_at > (CURDATE() - INTERVAL 5 DAY)
+group by created_date
+

_sql/analytics/w2l/w2l_results.sql

@@ -0,0 +1,42 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Histogram of results
+select
+    date(created_at) as created_date,
+    count(*) as result_count
+from w2l_results
+group by created_date
+
+-- Histogram of results and durations
+select
+    date(created_at) as created_date,
+    sum(duration_millis) / 1000 / 60 / 60 as hours,
+    avg(duration_millis) / 60 as average_seconds,
+    count(*) as video_count
+from w2l_results
+group by created_date
+
+-- Histogram of results (within range)
+select
+    date(created_at) as created_date,
+    count(*) as result_count
+from w2l_results
+where created_at > (CURDATE() - INTERVAL 5 DAY)
+group by created_date
+
+-- Created by date (within range)
+select
+    date(created_at) as created_date,
+    count(*) as use_count
+from w2l_results
+where date(created_at) > "2023-07-31"
+group by created_date
+
+select
+    date(created_at) as created_date,
+    count(*) as created_count
+from tts_inference_jobs
+where created_at > (CURDATE() - INTERVAL 90 DAY)
+group by created_date

_sql/analytics/w2l/w2l_templates.sql

@@ -0,0 +1,19 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Histogram of template contributions
+select
+    date(created_at) as created_date,
+    count(*) as template_count
+from w2l_templates
+group by created_date
+
+-- Histogram of template contributions (within range)
+select
+    date(created_at) as created_date,
+    count(*) as template_count
+from w2l_templates
+where created_at > (CURDATE() - INTERVAL 5 DAY)
+group by created_date
+

_sql/migrations/2023-07-18-202622_alter_voice_conversion_models_add_index_file/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE voice_conversion_models DROP COLUMN has_index_file;

_sql/migrations/2023-07-18-202622_alter_voice_conversion_models_add_index_file/up.sql

@@ -0,0 +1,7 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE voice_conversion_models
+    ADD COLUMN has_index_file BOOLEAN NOT NULL DEFAULT FALSE
+    AFTER ietf_primary_language_subtag;

_sql/migrations/2023-08-24-002715_media_files/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+drop table media_files;

_sql/migrations/2023-08-24-002715_media_files/up.sql

@@ -0,0 +1,159 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- This table contains both user uploads and inference or processing results.
+CREATE TABLE media_files (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Effective "primary key" (PUBLIC)
+  token VARCHAR(32) NOT NULL,
+
+  -- We don't need an idempotency token!
+  -- Store upload idempotency tokens in a separate table.
+  -- uuid_idempotency_token VARCHAR(36) NOT NULL,
+
+  -- ========== FOREIGN KEY TO ORIGIN ==========
+
+  -- Where the file came from:
+  --   * 'inference' for inference output
+  --   * 'processed' for processed file
+  --   * 'upload' for direct user upload
+  origin_category VARCHAR(16) NOT NULL,
+
+  -- For inference that can be tied back to a model, the type of model.
+  --   * 'sad_talker', which doesn't have a value for maybe_origin_model_token (!!!)
+  --   * (more tome come)
+  maybe_origin_model_type VARCHAR(32) DEFAULT NULL,
+
+  -- For inference that can be tied back to a model, the token of the model.
+  -- For zero shot models, this may be set as the token for the vector.
+  maybe_origin_model_token VARCHAR(32) DEFAULT NULL,
+
+  -- The original filename of the media (if uploaded by a user)
+  maybe_origin_filename VARCHAR(255) DEFAULT NULL,
+
+  -- ========== METADATA ==========
+
+  -- We can store text transcripts, etc. in other tables and refer back via foreign key.
+  -- Titles and even descriptions can go in another table(s): media_file_annotations, media_file_tags, media_file_descriptions, ...
+  -- (Not everything would have a text transcript.)
+
+  -- ========== MEDIA DETAILS ==========
+
+  -- Type of media:
+  --   * 'audio' for wav, mp3, etc.
+  --   * 'image' for a variety of video types.
+  --   * 'video' for a variety of video types.
+  media_type VARCHAR(16) NOT NULL,
+
+  -- The file's mime type.
+  maybe_mime_type VARCHAR(32) DEFAULT NULL,
+
+  -- The file size.
+  file_size_bytes INT(10) NOT NULL DEFAULT 0,
+
+  -- The duration in millis, if audio or video.
+  maybe_duration_millis INT(10) DEFAULT NULL,
+
+  -- Audio encoder details
+  -- Only present if the file was audio or a video that had audio.
+  maybe_audio_encoding VARCHAR(32) DEFAULT NULL,
+
+  -- Video encoder details
+  -- Only present if the file was a video.
+  maybe_video_encoding VARCHAR(32) DEFAULT NULL,
+
+  -- For videos, the original frame width.
+  maybe_frame_width INT(5) DEFAULT NULL,
+
+  -- For videos, the original frame height.
+  maybe_frame_height INT(5) DEFAULT NULL,
+
+  -- Checksum of the original media
+  -- SHA1 hash [SHA2 = CHAR(64), SHA1 = CHAR(40), MD5 = CHAR(32)]
+  -- checksum_sha1 CHAR(40) NOT NULL,
+
+  -- Checksum of the original media
+  -- SHA2 hash [SHA2 = CHAR(64), SHA1 = CHAR(40), MD5 = CHAR(32)]
+  -- Note that SHA2 is a hash family (SHA-228, SHA-256, SHA-384, SHA-512, SHA-512/224, SHA-512/256, ...)
+  -- and produces different lengths output depending on the choice of algorithm.
+  checksum_sha2 CHAR(64) NOT NULL,
+
+  -- ========== UPLOAD, TRANSCODING, AND TRUNCATION DETAILS ==========
+
+  -- The hash for the bucket directory that contains the original upload
+  -- as well as any transcodings, downsamplings, etc.
+  public_bucket_directory_hash  VARCHAR(32) NOT NULL,
+
+  -- The directory this media is uploaded to will be exclusive for this file.
+  -- Only this given record will live in this bucket, but the directory may include
+  -- other transcodings or truncations.
+  public_bucket_directory_full_path VARCHAR(255) NOT NULL,
+
+  -- We'll likely transcode (and potentially truncate) most media given to us.
+  -- This will store a json-encoded struct that details the changes.
+  -- TEXT = 65,535 bytes (64 KiB), ~= 4 bytes per UTF-8 character, ~ 16383 characters.
+  extra_file_modification_info TEXT DEFAULT NULL,
+
+  -- ========== CREATOR DETAILS AND PREFERENCES ==========
+
+  -- Foreign key to user
+  maybe_creator_user_token VARCHAR(32) DEFAULT NULL,
+
+  -- Based on a cookie sent by the frontend.
+  -- We'll save this even if the user is logged in.
+  maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL,
+
+  -- For abuse tracking.
+  -- Wide enough for IPv4/6
+  creator_ip_address VARCHAR(40) NOT NULL,
+
+  -- The creator can set a desired visibility for their data.
+  -- This does not always apply to every upload type.
+  -- Additionally, some upload types may require moderator approval prior
+  -- to being publicly listed, and this field has no bearing on that.
+  -- NB: DO NOT CHANGE THE ORDER OF THE ENUM VALUES DURING SCHEMA MIGRATIONS.
+  creator_set_visibility ENUM(
+    'public',
+    'hidden',
+    'private'
+  ) NOT NULL DEFAULT 'public',
+
+  -- The synthetic id associated with this result.
+  -- These ids are incremented on a per-user basis to help users
+  -- sequence their own work. They serve no other purpose.
+  maybe_creator_synthetic_id BIGINT(20) DEFAULT NULL,
+
+  -- ========== MODERATION DETAILS ==========
+
+  -- The last moderator that made changes last.
+  -- We should also build an audit table to supersede this.
+  maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
+
+  -- ========== TIMESTAMPS ==========
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- If this is removed by a mod or the creator.
+  user_deleted_at TIMESTAMP NULL,
+  mod_deleted_at TIMESTAMP NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (token),
+  KEY index_origin_category (origin_category),
+  KEY index_maybe_origin_model_type (maybe_origin_model_type),
+  KEY fk_maybe_origin_model_token (maybe_origin_model_token),
+  KEY fk_maybe_origin_model_type_and_token (maybe_origin_model_type, maybe_origin_model_token),
+  KEY index_media_type (media_type),
+  KEY index_checksum_sha2 (checksum_sha2),
+  KEY fk_maybe_creator_user_token (maybe_creator_user_token),
+  KEY fk_maybe_mod_user_token (maybe_mod_user_token),
+  KEY index_creator_ip_address (creator_ip_address),
+  KEY index_user_deleted_at (user_deleted_at),
+  KEY index_mod_deleted_at (mod_deleted_at)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_sql/migrations/2023-09-15-053054_generic_synthetic_ids/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+drop table generic_synthetic_ids;

_sql/migrations/2023-09-15-053054_generic_synthetic_ids/up.sql

@@ -0,0 +1,31 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- It gets unwieldy to show results without a numeric sequence.
+-- We create a synthetic sequence for each user to make revisiting results easier.
+-- This should be a reusable table for handling this for many different types of results.
+CREATE TABLE generic_synthetic_ids (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Effective "primary key" (PUBLIC)
+  user_token VARCHAR(32) NOT NULL,
+
+  -- The type of synthetic id to increment.
+  -- eg "media_file", "lipsync_animation", etc.
+  id_category VARCHAR(32) NOT NULL,
+
+  -- The next result id.
+  -- After we use an id, we increment this counter.
+  next_id BIGINT(20) NOT NULL DEFAULT 0,
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (user_token, id_category),
+  KEY fk_user_token (user_token)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_sql/migrations/2023-09-15-055106_alter_media_files_synthetic_ids/down.sql

@@ -0,0 +1,9 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Revert the rename
+ALTER TABLE media_files RENAME COLUMN maybe_creator_file_synthetic_id TO maybe_creator_synthetic_id ;
+
+ALTER TABLE media_files
+DROP COLUMN maybe_creator_category_synthetic_id;

_sql/migrations/2023-09-15-055106_alter_media_files_synthetic_ids/up.sql

@@ -0,0 +1,11 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- We'll have one ordering by files
+ALTER TABLE media_files RENAME COLUMN maybe_creator_synthetic_id TO maybe_creator_file_synthetic_id;
+
+-- And another ordering by category
+ALTER TABLE media_files
+    ADD COLUMN maybe_creator_category_synthetic_id BIGINT(20) DEFAULT NULL
+    AFTER maybe_creator_file_synthetic_id;

_sql/migrations/2023-09-15-070905_alter_media_files_drop_directory_full_path/down.sql

@@ -0,0 +1,7 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE media_files
+    ADD COLUMN public_bucket_directory_full_path VARCHAR(255) NOT NULL
+    AFTER public_bucket_directory_hash;

_sql/migrations/2023-09-15-070905_alter_media_files_drop_directory_full_path/up.sql

@@ -0,0 +1,7 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Kill the column
+ALTER TABLE media_files
+DROP COLUMN public_bucket_directory_full_path;

_sql/migrations/2023-09-15-083401_alter_media_files_add_workers/down.sql

@@ -0,0 +1,7 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE media_files DROP COLUMN generated_by_cluster;
+ALTER TABLE media_files DROP COLUMN generated_by_worker;
+ALTER TABLE media_files DROP COLUMN is_generated_on_prem;

_sql/migrations/2023-09-15-083401_alter_media_files_add_workers/up.sql

@@ -0,0 +1,15 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE media_files
+    ADD COLUMN is_generated_on_prem BOOLEAN NOT NULL DEFAULT FALSE
+    AFTER maybe_mod_user_token;
+
+ALTER TABLE media_files
+    ADD COLUMN generated_by_worker VARCHAR(255) DEFAULT NULL
+    AFTER is_generated_on_prem;
+
+ALTER TABLE media_files
+    ADD COLUMN generated_by_cluster VARCHAR(255) DEFAULT NULL
+    AFTER generated_by_worker;

_sql/migrations_squashed/generic_inference_jobs.sql

@@ -26,6 +26,13 @@ CREATE TABLE generic_inference_jobs (
   -- Potential part of the composite foreign key to the primary model being used, if any.
   -- This will normally live in `maybe_inference_args`, but in this case, it's useful for
   -- running easy database analytical queries.
+  --
+  -- Examples (may not be up to date):
+  --    * rvc_v2
+  --    * so_vits_svc
+  --    * tacotron2
+  --    * vits
+  --
   maybe_model_type VARCHAR(32) DEFAULT NULL,
 
   -- Potential part of the composite foreign key to the primary model being used, if any.

_sql/migrations_squashed/media_files.sql

@@ -0,0 +1,181 @@
+-- NB: This is a manually squashed view of all the CREATE and ALTER statements,
+-- with comments attached to the fields for centralized documentation.
+
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- This table contains both user uploads and inference or processing results.
+CREATE TABLE media_files (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Effective "primary key" (PUBLIC)
+  token VARCHAR(32) NOT NULL,
+
+  -- We don't need an idempotency token!
+  -- Store upload idempotency tokens in a separate table.
+  -- uuid_idempotency_token VARCHAR(36) NOT NULL,
+
+  -- ========== FOREIGN KEY TO ORIGIN ==========
+
+  -- Where the file came from:
+  --   * 'inference' for inference output
+  --   * 'processed' for processed file
+  --   * 'upload' for direct user upload
+  origin_category VARCHAR(16) NOT NULL,
+
+  -- For inference that can be tied back to a model, the type of model.
+  --   * 'sad_talker', which doesn't have a value for maybe_origin_model_token (!!!)
+  --   * (more tome come)
+  maybe_origin_model_type VARCHAR(32) DEFAULT NULL,
+
+  -- For inference that can be tied back to a model, the token of the model.
+  -- For zero shot models, this may be set as the token for the vector.
+  maybe_origin_model_token VARCHAR(32) DEFAULT NULL,
+
+  -- The original filename of the media (if uploaded by a user)
+  maybe_origin_filename VARCHAR(255) DEFAULT NULL,
+
+  -- ========== METADATA ==========
+
+  -- We can store text transcripts, etc. in other tables and refer back via foreign key.
+  -- Titles and even descriptions can go in another table(s): media_file_annotations, media_file_tags, media_file_descriptions, ...
+  -- (Not everything would have a text transcript.)
+
+  -- ========== MEDIA DETAILS ==========
+
+  -- Type of media:
+  --   * 'audio' for wav, mp3, etc.
+  --   * 'image' for a variety of video types.
+  --   * 'video' for a variety of video types.
+  media_type VARCHAR(16) NOT NULL,
+
+  -- The file's mime type.
+  maybe_mime_type VARCHAR(32) DEFAULT NULL,
+
+  -- The file size.
+  file_size_bytes INT(10) NOT NULL DEFAULT 0,
+
+  -- The duration in millis, if audio or video.
+  maybe_duration_millis INT(10) DEFAULT NULL,
+
+  -- Audio encoder details
+  -- Only present if the file was audio or a video that had audio.
+  maybe_audio_encoding VARCHAR(32) DEFAULT NULL,
+
+  -- Video encoder details
+  -- Only present if the file was a video.
+  maybe_video_encoding VARCHAR(32) DEFAULT NULL,
+
+  -- For videos, the original frame width.
+  maybe_frame_width INT(5) DEFAULT NULL,
+
+  -- For videos, the original frame height.
+  maybe_frame_height INT(5) DEFAULT NULL,
+
+  -- Checksum of the original media
+  -- SHA1 hash [SHA2 = CHAR(64), SHA1 = CHAR(40), MD5 = CHAR(32)]
+  -- checksum_sha1 CHAR(40) NOT NULL,
+
+  -- Checksum of the original media
+  -- SHA2 hash [SHA2 = CHAR(64), SHA1 = CHAR(40), MD5 = CHAR(32)]
+  -- Note that SHA2 is a hash family (SHA-228, SHA-256, SHA-384, SHA-512, SHA-512/224, SHA-512/256, ...)
+  -- and produces different lengths output depending on the choice of algorithm.
+  checksum_sha2 CHAR(64) NOT NULL,
+
+  -- ========== UPLOAD, TRANSCODING, AND TRUNCATION DETAILS ==========
+
+  -- The hash for the bucket directory that contains the original upload
+  -- as well as any transcodings, downsamplings, etc.
+  public_bucket_directory_hash  VARCHAR(32) NOT NULL,
+
+  -- NB: Removed, since this can be derived.
+  -- The directory this media is uploaded to will be exclusive for this file.
+  -- Only this given record will live in this bucket, but the directory may include
+  -- other transcodings or truncations.
+  -- public_bucket_directory_full_path VARCHAR(255) NOT NULL,
+
+  -- We'll likely transcode (and potentially truncate) most media given to us.
+  -- This will store a json-encoded struct that details the changes.
+  -- TEXT = 65,535 bytes (64 KiB), ~= 4 bytes per UTF-8 character, ~ 16383 characters.
+  extra_file_modification_info TEXT DEFAULT NULL,
+
+  -- ========== CREATOR DETAILS AND PREFERENCES ==========
+
+  -- Foreign key to user
+  maybe_creator_user_token VARCHAR(32) DEFAULT NULL,
+
+  -- Based on a cookie sent by the frontend.
+  -- We'll save this even if the user is logged in.
+  maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL,
+
+  -- For abuse tracking.
+  -- Wide enough for IPv4/6
+  creator_ip_address VARCHAR(40) NOT NULL,
+
+  -- The creator can set a desired visibility for their data.
+  -- This does not always apply to every upload type.
+  -- Additionally, some upload types may require moderator approval prior
+  -- to being publicly listed, and this field has no bearing on that.
+  -- NB: DO NOT CHANGE THE ORDER OF THE ENUM VALUES DURING SCHEMA MIGRATIONS.
+  creator_set_visibility ENUM(
+    'public',
+    'hidden',
+    'private'
+  ) NOT NULL DEFAULT 'public',
+
+  -- The synthetic id associated with this result.
+  -- These ids are incremented on a per-user basis to help users
+  -- sequence their own work. They serve no other purpose.
+  -- "file" synthetic ids are incremented over all media files
+  maybe_creator_file_synthetic_id BIGINT(20) DEFAULT NULL,
+
+  -- "category" synthetic ids are incremented over a category type
+  maybe_creator_category_synthetic_id BIGINT(20) DEFAULT NULL,
+
+  -- ========== MODERATION DETAILS ==========
+
+  -- The last moderator that made changes last.
+  -- We should also build an audit table to supersede this.
+  maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
+
+  -- ========== INFERENCE METADATA, DC, ROUTING, DEBUGGING ==========
+
+  -- For inference and processing, whether we generated this result from
+  -- our own data center (vs. cloud).
+  is_generated_on_prem BOOLEAN NOT NULL DEFAULT FALSE,
+
+  -- For inference or processing, which worker handled the processing.
+  -- Worker hostname (linux hostname, k8s pod name)
+  generated_by_worker VARCHAR(255) DEFAULT NULL,
+
+  -- For inference or processing, which cluster handled the processing.
+  -- Cluster name (k8s)
+  generated_by_cluster VARCHAR(255) DEFAULT NULL,
+
+  -- ========== TIMESTAMPS ==========
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- If this is removed by a mod or the creator.
+  user_deleted_at TIMESTAMP NULL,
+  mod_deleted_at TIMESTAMP NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (token),
+  KEY index_origin_category (origin_category),
+  KEY index_maybe_origin_model_type (maybe_origin_model_type),
+  KEY fk_maybe_origin_model_token (maybe_origin_model_token),
+  KEY fk_maybe_origin_model_type_and_token (maybe_origin_model_type, maybe_origin_model_token),
+  KEY index_media_type (media_type),
+  KEY index_checksum_sha2 (checksum_sha2),
+  KEY fk_maybe_creator_user_token (maybe_creator_user_token),
+  KEY fk_maybe_mod_user_token (maybe_mod_user_token),
+  KEY index_creator_ip_address (creator_ip_address),
+  KEY index_user_deleted_at (user_deleted_at),
+  KEY index_mod_deleted_at (mod_deleted_at)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_sql/migrations_squashed/voice_conversion_models.sql

@@ -39,6 +39,12 @@ CREATE TABLE voice_conversion_models (
   -- (Not that it matters apart from categorization, since voice conversion is universal.)
   ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en',
 
+  -- ========== EXTRA MODEL INFO ==========
+
+  -- RVC (v2) specific - whether the model has an index file associated with it
+  -- These files improve the quality of the results.
+  has_index_file BOOLEAN NOT NULL DEFAULT FALSE,
+
   -- ========== MODEL PLACEMENT ==========
 
   -- Whether the voice features on the front of FakeYou

codegen_database.sh

@@ -32,5 +32,9 @@ build_shared_sqlite_database_library() {
   popd
 }
 
+echo 'mysql prepare'
 build_shared_mysql_database_library
+
+echo 'sqlite prepare'
 build_shared_sqlite_database_library
+

crates/_types/buckets/src/public/media_files/directory.rs

@@ -0,0 +1,75 @@
+use crate::public::public_path::PublicPath;
+use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string;
+use crockford::crockford_entropy_lower;
+
+// TODO: Generate these from a macro.
+
+// TODO: Use a central path registry for quick reference
+const MEDIA_FILE_DIRECTORY: &str = "/media";
+
+/// Directory for user media uploads.
+/// Each uploaded file gets its own directory so that we can store the original
+/// file alongside re-processed derivatives.
+#[derive(Clone)]
+pub struct MediaFileBucketDirectory {
+  object_hash: String,
+  directory: String,
+}
+
+impl PublicPath for MediaFileBucketDirectory {}
+
+impl MediaFileBucketDirectory {
+
+  pub fn generate_new() -> Self {
+    let entropy = crockford_entropy_lower(32);
+    Self::from_object_hash(&entropy)
+  }
+
+  pub fn from_object_hash(object_hash: &str) -> Self {
+    // TODO: Path construction could be cleaner.
+    let middle = hashed_directory_path_long_string(object_hash);
+    let directory = format!("{}/{}{}", MEDIA_FILE_DIRECTORY, middle, object_hash);
+    Self {
+      object_hash: object_hash.to_string(),
+      directory,
+    }
+  }
+
+  pub fn get_directory_path_str(&self) -> &str {
+    &self.directory
+  }
+
+  pub fn get_object_hash(&self) -> &str {
+    &self.object_hash
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::public::media_files::directory::MediaFileBucketDirectory;
+
+  #[test]
+  pub fn generate_new_entropy() {
+    let directory = MediaFileBucketDirectory::generate_new();
+    assert_eq!(directory.get_object_hash().len(), 32);
+  }
+
+  #[test]
+  pub fn get_directory_path_str() {
+    let directory = MediaFileBucketDirectory::from_object_hash("abcdefghijk");
+    assert_eq!(directory.get_directory_path_str(), "/media/a/b/c/d/e/abcdefghijk");
+  }
+
+  #[test]
+  pub fn get_directory_path_str_short_name() {
+    let directory = MediaFileBucketDirectory::from_object_hash("foo");
+    assert_eq!(directory.get_directory_path_str(), "/media/f/o/foo");
+  }
+
+  #[test]
+  pub fn get_object_hash() {
+    let hash = "abcdefghijk";
+    let directory = MediaFileBucketDirectory::from_object_hash(hash);
+    assert_eq!(directory.get_object_hash(), hash);
+  }
+}

crates/_types/buckets/src/public/media_files/mod.rs

@@ -0,0 +1,2 @@
+pub mod directory;
+pub mod original_file;

crates/_types/buckets/src/public/media_files/original_file.rs

@@ -0,0 +1,101 @@
+use crate::public::media_files::directory::MediaFileBucketDirectory;
+use crate::public::public_path::PublicPath;
+use crockford::crockford_entropy_lower;
+use std::path::PathBuf;
+
+const ORIGINAL_FILE_BASENAME : &str = "file.bin";
+
+// TODO: Generate these from a macro.
+
+/// The original user upload file.
+/// It may have derivative files (down samples, crops, etc.) that live alongside it.
+#[derive(Clone)]
+pub struct MediaFileBucketPath {
+  directory: MediaFileBucketDirectory,
+  full_object_path: String,
+}
+
+impl PublicPath for MediaFileBucketPath {}
+
+impl MediaFileBucketPath {
+
+  pub fn generate_new() -> Self {
+    let entropy = crockford_entropy_lower(32);
+    Self::from_object_hash(&entropy)
+  }
+
+  pub fn from_object_hash(hash: &str) -> Self {
+    // TODO: Path construction could be cleaner.
+    let directory = MediaFileBucketDirectory::from_object_hash(hash);
+    let full_object_path = format!("{}/{}", directory.get_directory_path_str(), ORIGINAL_FILE_BASENAME);
+    Self {
+      directory,
+      full_object_path,
+    }
+  }
+
+  pub fn get_full_object_path_str(&self) -> &str {
+    &self.full_object_path
+  }
+
+  pub fn to_full_object_pathbuf(&self) -> PathBuf {
+    PathBuf::from(&self.full_object_path)
+  }
+
+  pub fn get_directory(&self) -> &MediaFileBucketDirectory {
+    &self.directory
+  }
+
+  pub fn get_object_hash(&self) -> &str {
+    &self.directory.get_object_hash()
+  }
+
+  pub fn get_basename() -> &'static str {
+    ORIGINAL_FILE_BASENAME
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::public::media_files::original_file::MediaFileBucketPath;
+  use std::path::PathBuf;
+
+  #[test]
+  pub fn generate_new_entropy() {
+    let file = MediaFileBucketPath::generate_new();
+    assert_eq!(file.get_object_hash().len(), 32);
+    assert_eq!(file.get_directory().get_object_hash().len(), 32);
+  }
+
+  #[test]
+  pub fn get_full_object_path_str() {
+    let file = MediaFileBucketPath::from_object_hash("abcdefghijk");
+    assert_eq!(file.get_full_object_path_str(), "/media/a/b/c/d/e/abcdefghijk/file.bin");
+  }
+
+  #[test]
+  pub fn to_full_object_pathbuf() {
+    let file = MediaFileBucketPath::from_object_hash("abcdefghijk");
+    assert_eq!(file.to_full_object_pathbuf(), PathBuf::from("/media/a/b/c/d/e/abcdefghijk/file.bin"));
+  }
+
+  #[test]
+  pub fn get_full_object_path_str_short_name() {
+    let file = MediaFileBucketPath::from_object_hash("foo");
+    assert_eq!(file.get_full_object_path_str(), "/media/f/o/foo/file.bin");
+  }
+
+  #[test]
+  pub fn get_full_object_path_str_starts_with_directory() {
+    let file = MediaFileBucketPath::from_object_hash("abcdefghijk");
+    let full_path = file.get_full_object_path_str();
+    assert!(full_path.starts_with(file.get_directory().get_directory_path_str()))
+  }
+
+  #[test]
+  pub fn get_object_hash() {
+    let hash = "abcdefghijk";
+    let file = MediaFileBucketPath::from_object_hash(hash);
+    assert_eq!(file.get_object_hash(), hash);
+  }
+}

crates/_types/buckets/src/public/media_uploads/directory.rs

@@ -5,7 +5,7 @@ use crockford::crockford_entropy_lower;
 // TODO: Generate these from a macro.
 
 // TODO: Use a central path registry for quick reference
-const MEDIA_UPLOAD_DIRECTORY : &'static str = "/media_upload";
+const MEDIA_UPLOAD_DIRECTORY : &str = "/media_upload";
 
 /// Directory for user media uploads.
 /// Each uploaded file gets its own directory so that we can store the original

crates/_types/buckets/src/public/media_uploads/original_file.rs

@@ -3,7 +3,7 @@ use crate::public::public_path::PublicPath;
 use crockford::crockford_entropy_lower;
 use std::path::PathBuf;
 
-const ORIGINAL_FILE_BASENAME : &'static str = "original_upload.bin";
+const ORIGINAL_FILE_BASENAME : &str = "original_upload.bin";
 
 // TODO: Generate these from a macro.