Minor update

Sad Talker worker, file uploads, etc.

.dockerignore

@@ -1,9 +1,11 @@
 *secret*sh
 *~
+.dockerignore
 .env
 .env-secrets
 .git
 .idea/
+Dockerfile
 __pycache__/
 _ignored_schema.rs
 models/Wav2Lip/python/

.env

@@ -1,16 +1,27 @@
+SERVER_ENVIRONMENT=development
+
 DATABASE_URL=mysql://storyteller:password@localhost/storyteller
 #DATABASE_URL=mysql://root:root@localhost/storyteller
 #DATABASE_URL=mysql://root@localhost/storyteller
 
+# Diesel-cli reads this for the migration directory to use.
+# We've renamed it so the migrations are sorted near the top of the project directory tree
+# (since they kind of serve as documentation)
+MIGRATION_DIRECTORY=_sql/migrations/
+
 # SQLX normally requires a live database to reflect upon at build time.
 # This setting turns that on and requires a `sql-data.json` cache file instead.
 # See https://github.com/launchbadge/sqlx/tree/master/sqlx-cli#readme
 #SQLX_OFFLINE=true
 
-# We can't serve HTTPS locally without self-signed certificate nonsense.
-COOKIE_SECURE=false
-COOKIE_DOMAIN=.jungle.horse
-WEBSITE_HOMEPAGE_REDIRECT=http://jungle.horse
+# NB(bt, 2023-03-20):
+#  These are used in storyteller-web, image-generation-service, and websocket-gateway.
+#  I'm moving them to env configs that live under storyteller-web/.env.
+#  This will break the other services for now.
+# # We can't serve HTTPS locally without self-signed certificate nonsense.
+# COOKIE_SECURE=false
+# COOKIE_DOMAIN=.jungle.horse
+# WEBSITE_HOMEPAGE_REDIRECT=http://jungle.horse
 
 # Bucket config (non-secret)
 REGION_NAME=us-east1

.gitattributes

@@ -0,0 +1,5 @@
+# Ignore certain code for Github language statistics
+# https://github.com/github/linguist/blob/master/docs/overrides.md
+old_code/trumpet-trumped.com/dictionary/cmudict-0.7b linguist-vendored
+old_code/voder-vocodes-1.0/frontends/admin/node_modules linguist-vendored
+windows/sqlite linguist-vendored

.github/workflows/docker-publish-cpu.yml

@@ -0,0 +1,115 @@
+# Adapted from
+# https://github.com/actions/starter-workflows/ci/docker-publish.yml
+# If using base images, don't forget to add READ access, eg. from a package settings page like:
+# https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access
+name: "[CPU] Build GHCR Docker artifact"
+on:
+  push:
+    branches:
+      # NB: Default-branch doesn't work, despite Github's documentation.
+      #- $default-branch
+      - master
+env:
+  IMAGE_NAME: storyteller-rust
+
+jobs:
+  # Push image to GitHub Packages.
+  # See also https://docs.docker.com/docker-hub/builds/
+  # See also https://depot.dev/blog/docker-layer-caching-in-github-actions
+  # See also https://evilmartians.com/chronicles/build-images-on-github-actions-with-docker-layer-caching
+  build-docker-image:
+    name: "Build Docker Image"
+    #runs-on: ubuntu-22.04 # runs-on: ubuntu-latest
+    #runs-on: github-ubuntu-16core # NB: Premium GitHub runner with more cores
+    runs-on: github-ubuntu-64core # NB: Premium GitHub runner with more cores
+    if: github.event_name == 'push'
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v3
+
+      # - uses: docker/setup-buildx-action@v1
+      #   name: Set up Docker Buildx
+
+      - uses: docker/login-action@v1
+        name: Login to GitHub Container Registry
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }} # username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Calculate short SHA
+        id: vars
+        # echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+        run: |
+          echo "sha_short=$(echo ${{ github.sha }} | cut -c1-12)" >> $GITHUB_OUTPUT
+
+      - name: Emit short SHA
+        run: |
+          echo "Full SHA: ${{ github.sha }}"
+          echo "Short SHA: ${{ steps.vars.outputs.sha_short }}"
+
+      #- uses: actions/cache@v2
+      #  name: Cache Docker layers
+      #  with:
+      #    path: /tmp/.buildx-cache
+      #    key: ${{ runner.os }}-buildx-${{ github.sha }}
+      #    restore-keys: |
+      #      ${{ runner.os }}-buildx-
+
+      # Docs for this action are at https://github.com/marketplace/actions/build-docker-images-using-cache
+      - uses: whoan/docker-build-with-cache-action@v6
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }} # username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          image_name: storytold/storyteller-rust
+          dockerfile: service_cpu.Dockerfile
+          # NB: The `git_push_tag: true` option does not work as expected, so we put the sha in `image_tag`:
+          # https://github.com/whoan/docker-build-with-cache-action/issues/72
+          image_tag: latest,${{ github.sha }},${{ steps.vars.outputs.sha_short }}
+          # NB: Inject the Git SHA into the container so the servers can emit their own version:
+          build_extra_args: "--build-arg=GIT_SHA=${{ github.sha }}"
+
+      # - name: Build image
+      #   run: |
+      #     SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
+      #     docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" --build-arg GIT_SHA=${SHORT_SHA}
+
+      # - name: Push image to GitHub Container Registry
+      #   run: |
+      #     IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+
+      #     # Change all uppercase to lowercase
+      #     IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+
+      #     # Strip git ref prefix from version
+      #     VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+
+      #     # Strip "v" prefix from tag name
+      #     [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+
+      #     # Use Docker `latest` tag convention
+      #     [ "$VERSION" == "$default-branch" ] && VERSION=latest
+
+      #     echo IMAGE_ID=$IMAGE_ID
+      #     echo VERSION=$VERSION
+
+      #     docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
+      #     docker push $IMAGE_ID:$VERSION
+
+      #     SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
+
+      #     docker tag $IMAGE_NAME $IMAGE_ID:$SHORT_SHA
+      #     docker push $IMAGE_ID:$SHORT_SHA
+
+      - name: Cleanup / Debug
+        # This job will run even on failures to help with debugging.
+        if: always()
+        run: |
+          echo "Report at final github action step:"
+          echo "Disk usage:"
+          pwd
+          du -hsc * | sort -hr

.github/workflows/docker-publish-gpu.yml

@@ -0,0 +1,115 @@
+# Adapted from
+# https://github.com/actions/starter-workflows/ci/docker-publish.yml
+# If using base images, don't forget to add READ access, eg. from a package settings page like:
+# https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access
+name: "[GPU] Build GHCR Docker artifact"
+on:
+  push:
+    branches:
+      # NB: Default-branch doesn't work, despite Github's documentation.
+      #- $default-branch
+      - master
+env:
+  IMAGE_NAME: storyteller-rust-gpu
+
+jobs:
+  # Push image to GitHub Packages.
+  # See also https://docs.docker.com/docker-hub/builds/
+  # See also https://depot.dev/blog/docker-layer-caching-in-github-actions
+  # See also https://evilmartians.com/chronicles/build-images-on-github-actions-with-docker-layer-caching
+  build-docker-image:
+    name: "Build Docker Image"
+    #runs-on: ubuntu-22.04 # runs-on: ubuntu-latest
+    #runs-on: github-ubuntu-16core # NB: Premium GitHub runner with more cores
+    runs-on: github-ubuntu-64core # NB: Premium GitHub runner with more cores
+    if: github.event_name == 'push'
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v3
+
+      # - uses: docker/setup-buildx-action@v1
+      #   name: Set up Docker Buildx
+
+      - uses: docker/login-action@v1
+        name: Login to GitHub Container Registry
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }} # username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Calculate short SHA
+        id: vars
+        # echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+        run: |
+          echo "sha_short=$(echo ${{ github.sha }} | cut -c1-12)" >> $GITHUB_OUTPUT
+
+      - name: Emit short SHA
+        run: |
+          echo "Full SHA: ${{ github.sha }}"
+          echo "Short SHA: ${{ steps.vars.outputs.sha_short }}"
+
+      #- uses: actions/cache@v2
+      #  name: Cache Docker layers
+      #  with:
+      #    path: /tmp/.buildx-cache
+      #    key: ${{ runner.os }}-buildx-${{ github.sha }}
+      #    restore-keys: |
+      #      ${{ runner.os }}-buildx-
+
+      # Docs for this action are at https://github.com/marketplace/actions/build-docker-images-using-cache
+      - uses: whoan/docker-build-with-cache-action@v6
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }} # username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          image_name: storytold/storyteller-rust-gpu
+          dockerfile: service_gpu.Dockerfile
+          # NB: The `git_push_tag: true` option does not work as expected, so we put the sha in `image_tag`:
+          # https://github.com/whoan/docker-build-with-cache-action/issues/72
+          image_tag: latest,${{ github.sha }},${{ steps.vars.outputs.sha_short }}
+          # NB: Inject the Git SHA into the container so the servers can emit their own version:
+          build_extra_args: "--build-arg=GIT_SHA=${{ github.sha }}"
+
+      # - name: Build image
+      #   run: |
+      #     SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
+      #     docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" --build-arg GIT_SHA=${SHORT_SHA}
+
+      # - name: Push image to GitHub Container Registry
+      #   run: |
+      #     IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+
+      #     # Change all uppercase to lowercase
+      #     IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+
+      #     # Strip git ref prefix from version
+      #     VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+
+      #     # Strip "v" prefix from tag name
+      #     [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+
+      #     # Use Docker `latest` tag convention
+      #     [ "$VERSION" == "$default-branch" ] && VERSION=latest
+
+      #     echo IMAGE_ID=$IMAGE_ID
+      #     echo VERSION=$VERSION
+
+      #     docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
+      #     docker push $IMAGE_ID:$VERSION
+
+      #     SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
+
+      #     docker tag $IMAGE_NAME $IMAGE_ID:$SHORT_SHA
+      #     docker push $IMAGE_ID:$SHORT_SHA
+
+      - name: Cleanup / Debug
+        # This job will run even on failures to help with debugging.
+        if: always()
+        run: |
+          echo "Report at final github action step:"
+          echo "Disk usage:"
+          pwd
+          du -hsc * | sort -hr

.github/workflows/docker-publish.yml

@@ -1,67 +0,0 @@
-# Adapted from
-# https://github.com/actions/starter-workflows/ci/docker-publish.yml
-# If using base images, don't forget to add READ access, eg. from a package settings page like:
-# https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access
-name: Publish Docker Container to GHCR
-on:
-  push:
-    branches:
-      # NB: Default-branch doesn't work, despite Github's documentation.
-      #- $default-branch
-      - master
-env:
-  IMAGE_NAME: storyteller-rust
-
-jobs:
-  # Push image to GitHub Packages.
-  # See also https://docs.docker.com/docker-hub/builds/
-  push:
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push'
-    permissions:
-      packages: write
-      contents: read
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          #username: ${{ github.repository_owner }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }} 
-
-      - name: Build image
-        run: |
-          SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
-          docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" --build-arg GIT_SHA=${SHORT_SHA}
-
-      - name: Push image to GitHub Container Registry
-        run: |
-          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
-
-          # Change all uppercase to lowercase
-          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
-
-          # Strip git ref prefix from version
-          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-
-          # Strip "v" prefix from tag name
-          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
-
-          # Use Docker `latest` tag convention
-          [ "$VERSION" == "$default-branch" ] && VERSION=latest
-
-          echo IMAGE_ID=$IMAGE_ID
-          echo VERSION=$VERSION
-
-          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
-          docker push $IMAGE_ID:$VERSION
-
-          SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
-
-          docker tag $IMAGE_NAME $IMAGE_ID:$SHORT_SHA
-          docker push $IMAGE_ID:$SHORT_SHA
-

.github/workflows_inactive/ci.yml

@@ -0,0 +1,52 @@
+name: CI
+on: [push, pull_request]
+env: 
+  MYSQL_URL: mysql://root:root@localhost/storyteller
+  DATABASE_URL: mysql://root:root@localhost/storyteller
+  REDIS_0_URL: redis://redisd@localhost/0
+  REDIS_1_URL: redis://redisd@localhost/1
+  DEBIAN_FRONTEND: noninteractive
+
+jobs:
+  setup_build_test:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+      - name: Cache cargo registry
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
+      - name: Setup mysql
+        run: |
+          sudo systemctl start mysql
+          sudo apt-get update
+          sudo apt-get -y install mysql-client libmysqlclient-dev
+          mysql -e "create user 'storyteller'@'localhost' IDENTIFIED BY 'password';" -uroot -proot 
+          mysql -e "create database storyteller; grant all on storyteller.* to 'storyteller'@'localhost';" -uroot -proot
+      - name: Setup Redis
+        run: |
+          sudo apt-get -y install redis-server
+          sudo service redis-server start
+      - name: Setup Rust
+        run: |
+          rustup set profile minimal
+          rustup show
+          rustup install 1.66
+          rustup default 1.66
+          rustup show
+          cargo install sqlx-cli --features openssl-vendored
+          cargo install diesel_cli
+      - name: Run database migrations
+        run: |
+          # nothing to see here ;)
+          set +e +o pipefail
+          diesel migration run
+          set -e -o pipefail
+      - name: Build 
+        run: cargo build --release
+      - name: Test
+        run: cargo test --release

.gitignore

@@ -1,22 +1,59 @@
-*.swo
-*.swp
+# Secrets are BAD!
 *secret*sh
-*~
 .env
+.env-aichatbot-secrets
 .env-secrets
-.idea/
-__pycache__/
-_ignored_schema.rs
 db.secrets
 db_secrets.toml
-models/Wav2Lip/python/
-models/tacotron2/python/
 production_database_secret_connect.sh
 production_database_secret_query.sh
 production_database_voice_clone_report_download.sh
 secrets.toml
 secrets.yaml
-target/
-tools/.secrets
-tools/db_secrets.toml
+_develop/tools/.secrets
+_develop/tools/db_secrets.toml
 twitch_secrets.toml
+crates/service/web/storyteller_web/config/storyteller-web.development-secrets.env
+
+# New env secrets (per-app)
+*-secrets.env
+
+# Rust build directory
+target/
+crates/lib/database_queries/target/
+crates/lib/sqlite_queries/target/
+
+# Vim noise
+*.swo
+*.swp
+*~
+
+# IntelliJ/CLion noise
+.idea/
+
+# Python noise
+__pycache__/
+
+# We use diesel for DB migrations only. We don't care about its codgen.
+_ignored_schema.rs
+
+# AI Chat Sidecar has its own schema managed by diesel, but we don't care for diesel codegen.
+crates/programs/aichatbot-sidecar/_ignored_schema.rs
+
+# AI Chat sidecar's SQlite database shouldn't be checked in
+crates/programs/aichatbot-sidecar/database.db
+
+# Runtime data for the AI chatbot
+runtime_data/
+
+# Old Python virtual environments
+models/Wav2Lip/python/
+models/tacotron2/python/
+
+# Stuff I want to keep around, but not commit
+scraps/
+
+# Windows build files
+# These should be built by the platform
+windows/sqlite/sqlite3.exp
+windows/sqlite/sqlite3.lib

.idea/dictionaries/bt.xml

@@ -1,18 +1,45 @@
 <component name="ProjectDictionaryState">
-  <!--
-  Despite spellcheck dictionaries being created per-user, IntelliJ loads all of them, per:
-  https://stackoverflow.com/a/28151692
-  -->
   <dictionary name="bt">
     <words>
       <w>arpabet</w>
       <w>backfill</w>
+      <w>cashapp</w>
       <w>cheermote</w>
+      <w>cidrs</w>
+      <w>crockford</w>
+      <w>culted</w>
+      <w>dalle</w>
+      <w>demuxer</w>
+      <w>dereferencing</w>
+      <w>downvote</w>
       <w>fakeyou</w>
+      <w>ffprobe</w>
+      <w>greenlight</w>
+      <w>greenlighting</w>
+      <w>greenlit</w>
+      <w>hanashi</w>
       <w>hifigan</w>
+      <w>hparams</w>
+      <w>huggingface</w>
+      <w>lipsync</w>
+      <w>pathbuf</w>
+      <w>replayable</w>
+      <w>retryably</w>
       <w>scrollback</w>
+      <w>softvc</w>
+      <w>sql's</w>
+      <w>symphonia</w>
+      <w>tacotron</w>
+      <w>techcrunch</w>
+      <w>transcoded</w>
+      <w>transcodings</w>
+      <w>unpermitted</w>
+      <w>upserting</w>
+      <w>vits</w>
+      <w>vocoded</w>
       <w>vocoder</w>
       <w>vocoders</w>
+      <w>vocodes</w>
     </words>
   </dictionary>
 </component>

Cargo.toml

@@ -1,29 +1,74 @@
 [workspace]
 
 members = [
+  # Types: common strongly-typed tokens, ids, payloads, etc.
+  # Also common cloud buckets and other "strongly typed" configs.
+  "crates/_types/buckets",
+  "crates/_types/enums",
+  "crates/_types/reusable_types", # TODO(bt, 2022-12-21): Deprecate
+  "crates/_types/tokens",
+
+  # CLI tools for operations, etc.
+  "crates/cli/database_backup",
+  "crates/cli/database_cleanup",
+  "crates/cli/scrape_test",
+
   # Shared libraries
-  "crates/lib/config",
-  "crates/lib/container_common",
-  "crates/lib/database_queries",
-  "crates/lib/google_drive_common",
-  "crates/lib/http_server_common",
+  "crates/lib/actix_helpers",
+  "crates/lib/api_clients/cloud_storage",
+  #"crates/lib/api_clients/fakeyou_client",
+  "crates/lib/api_clients/google_drive_common",
+  "crates/lib/api_clients/twitch_common",
+  "crates/lib/bootstrap",
+  "crates/lib/caching/memory_caching",
+  "crates/lib/caching/redis_caching",
+  "crates/lib/concurrency",
+  "crates/lib/crockford",
+  "crates/lib/datetimes",
+  "crates/lib/deprecated/config", # TODO(bt, 2023-02-13): Deprecate or repurpose
+  "crates/lib/deprecated/container_common", # TODO(bt, 2023-02-13): Pull apart and rehome pieces
+  "crates/lib/deprecated/http_server_common", # TODO(bt, 2023-03-20): Move useful pieces to actix_helpers
+  "crates/lib/envvar",
+  "crates/lib/errors",
+  "crates/lib/files/archives",
+  "crates/lib/files/filesys",
+  "crates/lib/files/hashing",
+  "crates/lib/files/media",
+  "crates/lib/files/mimetypes",
+  "crates/lib/idempotency",
   "crates/lib/jobs_common",
-  "crates/lib/memory_caching",
+  "crates/lib/mysql_queries",
   "crates/lib/redis_common",
-  "crates/lib/reusable_types",
-  "crates/lib/storage_buckets_common",
+  "crates/lib/server_environment",
+  #"crates/lib/sqlite_queries",
+  "crates/lib/sqlx_mysql_helpers",
+  "crates/lib/subprocess_common",
   "crates/lib/tts_common",
-  "crates/lib/twitch_common",
+  "crates/lib/url_config",
   "crates/lib/user_input_common",
+  "crates/lib/web_scrapers",
 
-  # Application 
-  "crates/service/generic_download_job",
-  "crates/service/storyteller_web",
-  "crates/service/tts_download_job",
-  "crates/service/tts_inference_job",
-  "crates/service/twitch_pubsub_subscriber",
-  "crates/service/w2l_download_job",
-  "crates/service/w2l_inference_job",
-  "crates/service/websocket_gateway",
+  # Programs: Desktop apps
+  #"crates/programs/aichatbot-sidecar",
+
+  # Services: API servers, jobs, workers, etc.
+  "crates/service/job/analytics_job",
+  "crates/service/job/download_job",
+  "crates/service/job/inference_job",
+  "crates/service/job/tts_download_job",
+  "crates/service/job/tts_inference_job",
+  "crates/service/job/w2l_download_job",
+  "crates/service/job/w2l_inference_job",
+  "crates/service/other/twitch_pubsub_subscriber",
+  "crates/service/web/dummy_service",
+  "crates/service/web/storyteller_web",
+  "crates/service/web/websocket_gateway",
+
+  # Service Plugins: complete subsystems that can be plugged into services (users, billing, etc.)
+  "crates/service/plugins/billing",
+  "crates/service/plugins/inference_workloads",
+  "crates/service/plugins/users",
+
+  # Vendored code
+  "crates/vendor/newrelic-telemetry-sdk-rust",
 ]
-

Cranky.toml

@@ -0,0 +1,58 @@
+# Persist clippy check configurations:
+# https://github.com/ericseppanen/cargo-cranky 
+# Clippy doesn't allow this (yet):
+# https://github.com/rust-lang/rust-clippy/issues/6625
+deny = [
+  # Rust
+  "unsafe_code", # We may remove this if needed
+
+  # Clippy - "serious" lints
+  "clippy::borrow_interior_mutable_const", # Seems dangerous to allow this. Eg. `const F = Lazy::new()`;
+
+  # Clippy - bad math
+  #"clippy::cast_possible_truncation", # Seems dangerous, eg f64 to u64.
+  #"clippy::cast_possible_wrap",
+
+  # Clippy - pointer safety / readability / correctness
+  "clippy::as_ptr_cast_mut",
+  "clippy::borrow_as_ptr",
+  "clippy::borrow_deref_ref",
+
+  # Clippy - performance
+  "clippy::cmp_owned",
+  "clippy::useless_vec",
+  "clippy::format_in_format_args",
+  #"clippy::redundant_clone",
+  #"clippy::unnecessary_to_owned",
+  #"clippy::vec_init_then_push",
+
+  # Clippy - "ergonomic" / "usage" lints
+  "clippy::option_option", # Lint: Don't ever Option<Option<T>>! Enums instead.
+  "clippy::zero_sized_map_values", # Lint: use a HashSet instead of HashMap<T, ()>.
+
+  # Clippy - "aesthetic" / "style" lints
+  "clippy::clone_double_ref", # Lint: feels safer/hygienic to disallow this, but I'm not strongly opposed.
+  "clippy::double_comparisons", # Lint: readability
+  "clippy::empty_structs_with_brackets", # Lint: {} feels less aesthetic than ;
+  "clippy::implicit_clone", # Lint: Recommendation here seems more idiomatic.
+  "clippy::needless_return", # Lint: not the worst offender, but let's try to keep unnecessary returns out.
+  "clippy::redundant_field_names", # Lint: Duplicate names is a burden when reading: you wonder why they're dupes.
+  "clippy::redundant_static_lifetimes", # Lint: feels more modern to elide static lifetimes
+
+  # Possible:
+  # "clippy::ptr_arg",
+  # "clippy::too_many_lines" (note: this is configurable)
+]
+
+warn = [
+]
+
+allow = [
+  # Rust
+  "deprecated", # Other systems can handle deprecated code instead of the linter
+  "missing_docs", # Not everything needs documentation. Too much documentation is bad. Literate code.
+
+  # Clippy
+  "clippy::if_same_then_else", # There are cases this is clearer! Branches may have notes / considerations.
+  "clippy::single_char_pattern", # This is like punishing trailing commas. Chars are dumb.
+]

Dockerfile

@@ -1,187 +0,0 @@
-# ==================== Python Build Base ====================
-
-FROM ghcr.io/storytold/docker-base-images-nvidia-cuda-experimental:080f96bc7087 as pybuild-base
-
-# The downloader needs youtube-dl as a python library
-# NB: No longer maintained: https://news.ycombinator.com/item?id=28319624
-#RUN pip3 install --upgrade youtube-dl
-
-# Used by the downloader script.
-# And it looks like youtube-dl is dead...
-# Per the README, it looks like pip will always pull the latest version.
-# https://github.com/yt-dlp/yt-dlp#update
-RUN python3 -m pip install --upgrade yt-dlp
-
-# ==================== Python Build Step 2: Wav2Lip Requirements ====================
-
-FROM pybuild-base as pybuild-requirements
-
-WORKDIR /
-
-COPY models/Wav2Lip ./models/Wav2Lip
-WORKDIR models/Wav2Lip
-
-# NB: We need Python3.6
-RUN python3.6 -m venv python
-#RUN virtualenv -p /usr/bin/python3.6  ... wait
-RUN . python/bin/activate \
-  && pip install --upgrade pip \
-  && pip install -r requirements.txt \
-  && deactivate
-
-WORKDIR /
-
-COPY models/tts ./models/tts
-WORKDIR models/tts
-
-# NB: Not sure if we need Python3.6 for Tacotron2, but Python3.8 gave me trouble.
-# NB: Setuptools fix: https://github.com/tensorflow/tensorflow/issues/34302#issuecomment-554450289
-RUN python3.6 -m venv python
-RUN . python/bin/activate \
-  && pip install --upgrade pip setuptools \
-  && pip install -r requirements-tacotron-python36.txt \
-  && deactivate
-
-# ==================== Rust Build Base ====================
-
-# Custom base image
-# Make sure to add this repository so it has read acces to the base image:
-# https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access
-FROM ghcr.io/storytold/docker-base-images-rust-ssl:latest as rust-build
-WORKDIR /tmp
-
-COPY Cargo.lock . 
-COPY Cargo.toml .
-COPY sqlx-data.json .
-COPY crates/ ./crates
-COPY db/ ./db
-
-RUN $HOME/.cargo/bin/cargo fetch
-
-ARG GIT_SHA
-RUN echo -n ${GIT_SHA} > GIT_SHA
-
-# Run all of the tests
-RUN SQLX_OFFLINE=true \
-  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
-  $HOME/.cargo/bin/cargo test
-
-# Build all the binaries.
-RUN SQLX_OFFLINE=true \
-  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
-  $HOME/.cargo/bin/cargo build \
-  --release \
-  --bin storyteller-web
-
-RUN SQLX_OFFLINE=true \
-  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
-  $HOME/.cargo/bin/cargo build \
-  --release \
-  --bin tts-download-job
-
-RUN SQLX_OFFLINE=true \
-  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
-  $HOME/.cargo/bin/cargo build \
-  --release \
-  --bin w2l-download-job
-
-RUN SQLX_OFFLINE=true \
-  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
-  $HOME/.cargo/bin/cargo build \
-  --release \
-  --bin generic-download-job
-
-RUN SQLX_OFFLINE=true \
-  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
-  $HOME/.cargo/bin/cargo build \
-  --release \
-  --bin tts-inference-job
-
-RUN SQLX_OFFLINE=true \
-  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
-  $HOME/.cargo/bin/cargo build \
-  --release \
-  --bin w2l-inference-job
-
-RUN SQLX_OFFLINE=true \
-  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
-  $HOME/.cargo/bin/cargo build \
-  --release \
-  --bin websocket-gateway
-
-RUN SQLX_OFFLINE=true \
-  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
-  $HOME/.cargo/bin/cargo build \
-  --release \
-  --bin twitch-pubsub-subscriber
-
-# Final image
-#  FROM ubuntu:xenial
-FROM pybuild-requirements as final
-
-WORKDIR /
-
-COPY scripts/ ./scripts
-
-# See: https://github.com/opencontainers/image-spec/blob/master/annotations.md
-LABEL org.opencontainers.image.authors='bt@brand.io, echelon@gmail.com'
-LABEL org.opencontainers.image.url='https://github.com/storytold/storyteller-web'
-LABEL org.opencontainers.image.documentation='https://github.com/storytold/storyteller-web'
-LABEL org.opencontainers.image.source='https://github.com/storytold/storyteller-web'
-
-# NB(echelon,2022-05-31): NVIDIA revoked some of their keys.
-# A better long term fix is to remove these dependencies outright and not have ML deps in this repo.
-# https://developer.nvidia.com/blog/updating-the-cuda-linux-gpg-repository-key/
-# https://forums.developer.nvidia.com/t/invalid-public-key-for-cuda-apt-repository/212901/11
-RUN apt-key del 7fa2af80
-
-# NB: Fix for 'E: Conflicting values set for option Signed-By regarding source'
-# NB: We need to remove all sources
-RUN rm $(find /etc/apt | grep cuda)
-
-#RUN wget https://developer.download.nvidia.com/compute/cuda/repos/$distro/$arch/cuda-keyring_1.0-1_all.deb
-RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb
-RUN dpkg -i cuda-keyring_1.0-1_all.deb
-
-# NB: Comment this out for non-debug images
-# TODO: Figure out how this is done elsewhere with just the static binaries
-# Others: `mysql-client`
-RUN apt-get update \
-    && apt-get install -y \
-        curl \
-        python3-pip \
-        python3 \
-        wget
-
-RUN python3 --version
-
-# TODO: Use venv to do this instead.
-RUN pip3 install gdown
-
-# Copy all the binaries.
-COPY --from=rust-build /tmp/target/release/storyteller-web /
-COPY --from=rust-build /tmp/target/release/tts-download-job /
-COPY --from=rust-build /tmp/target/release/tts-inference-job /
-COPY --from=rust-build /tmp/target/release/w2l-download-job /
-COPY --from=rust-build /tmp/target/release/w2l-inference-job /
-COPY --from=rust-build /tmp/target/release/websocket-gateway /
-COPY --from=rust-build /tmp/target/release/twitch-pubsub-subscriber /
-COPY --from=rust-build /tmp/target/release/generic-download-job /
-COPY --from=rust-build /tmp/GIT_SHA /
-
-# SSL certs are required for crypto
-COPY --from=rust-build /etc/ssl /etc/ssl
-
-# Required dynamically linked libraries
-COPY --from=rust-build /lib/x86_64-linux-gnu/libssl.*             /lib/x86_64-linux-gnu/
-COPY --from=rust-build /lib/x86_64-linux-gnu/libcrypto.*          /lib/x86_64-linux-gnu/
-
-# Make sure all the links resolve
-RUN ldd storyteller-web
-
-# Without a .env file, Rust crashes "mysteriously" (ugh)
-RUN touch .env
-RUN touch .env-secrets
-
-EXPOSE 8080
-CMD LD_LIBRARY_PATH=/usr/lib /storyteller-web

README.md

@@ -1,77 +1,49 @@
 storyteller-rust
 ================
 
-Our Rust monorepo, which services our 
-API surface area. 
-
-Application overview
---------------------
-
-* [FakeYou.com](https://fakeyou.com) is powered by:
-  * HTTP APIs:
-    * `storyteller-web` - HTTP + database monolith for FakeYou
-  * Asynchronous Jobs:
-    * `tts-download-job` - Async download of models
-    * `tts-inference-job` - Async TTS inference
-    * `w2l-download-job` - Async download of videos and images
-    * `w2l-inference-job` - Async W2L inference
-
-* [StorytellerStream](https://storyteller.stream) is powered by:
-  * HTTP APIs:
-    * `storyteller-web` - HTTP + database monolith for FakeYou
-  * Websocket events:
-    * `websocket-gateway` - Websocket to pull down Twitch events.
-  * Twitch API integration:
-    * `twitch-pubsub-subscriber` - Dynamically subscribes to Twitch PubSub for "active" users
-
-Schema and API notes
---------------------
-
-* [Redis Schema Notes README](_docs/redis_schema.md)
-
-Local development
------------------
-
-[Local development setup README](_docs/local_development_setup.md)
-
-Production and Deployment
--------------------------
-
-[Production builds and deployment README](_docs/production_builds_and_deployment.md)
-
-Development notes
------------------
-
-[Development notes README](_docs/development_notes.md)
-
-Dump a MySQL report
--------------------
+Our Rust monorepo, which serves both our API surface area and our downloadable client applications.
 
 ```
-mysql -u storyteller \
-  -pPasswordHere \
-  -h IP_HERE \
-  -P 3306 \
-  -D storyteller \
-  -B \
-  -e "select * from voice_clone_requests;" | sed "s/'/\'/;s/\t/\",\"/g;s/^/\"/;s/$/\"/;s/\n//g" > voice_clone_report.csv
+                 ."-,.__
+                 `.     `.  ,
+              .--'  .._,'"-' `.
+             .    .'         `'
+             `.   /          ,'
+               `  '--.   ,-"'
+                `"`   |  \
+                   -. \, |
+                    `--Y.'      ___.
+                         \     L._, \
+               _.,        `.   <  <\                _
+             ,' '           `, `.   | \            ( `
+          ../, `.            `  |    .\`.           \ \_
+         ,' ,..  .           _.,'    ||\l            )  '".
+        , ,'   \           ,'.-.`-._,'  |           .  _._`.
+      ,' /      \ \        `' ' `--/   | \          / /   ..\
+    .'  /        \ .         |\__ - _ ,'` `        / /     `.`.
+    |  '          ..         `-...-"  |  `-'      / /        . `.
+    | /           |L__           |    |          / /          `. `.
+   , /            .   .          |    |         / /             ` `
+  / /          ,. ,`._ `-_       |    |  _   ,-' /               ` \
+ / .           \"`_/. `-_ \_,.  ,'    +-' `-'  _,        ..,-.    \`.
+.  '         .-f    ,'   `    '.       \__.---'     _   .'   '     \ \
+' /          `.'    l     .' /          \..      ,_|/   `.  ,'`     L`
+|'      _.-""` `.    \ _,'  `            \ `.___`.'"`-.  , |   |    | \
+||    ,'      `. `.   '       _,...._        `  |    `/ '  |   '     .|
+||  ,'          `. ;.,.---' ,'       `.   `.. `-'  .-' /_ .'    ;_   ||
+|| '              V      / /           `   | `   ,'   ,' '.    !  `. ||
+||/            _,-------7 '              . |  `-'    l         /    `||
+. |          ,' .-   ,' ||               | .-.        `.      .'     ||
+ `'        ,'    `".'    |               |    `.        '. -.'       `'
+          /      ,'      |               |,'    \-.._,.'/'
+          .     /        .               .       \    .''
+        .`.    |         `.             /         :_,'.'
+          \ `...\   _     ,'-.        .'         /_.-'                       S T O R Y T E L L E R
+           `-.__ `,  `'   .  _.>----''.  _  __  /                        Turning memes into dreams
+                .'        /"'          |  "'   '_                      One Rust Monorepo at a time
+               /_|.-'\ ,".             '.'`__'-( \
+                 / ,"'"\,'               `/  `-.|" 
 ```
 
-TODO
-----
-
-* Revise this README to be more useful.
-
-Notes / TODOs:
-
-* Examples for good Actix+Sqlx Tests:
-  https://stackoverflow.com/questions/65370752/how-do-i-create-an-actix-web-server-that-accepts-both-sqlx-database-pools-and-tr
-
-* Actix/sqlx runtime compat:
-  https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734
-  
-* Redis caching of more endpoints
-
-* Jobs for analytics queries (#uses per model, etc)
-* Jobs for ingesting Discord, Twitter, Reddit RPAN chat, Twitch chat (not PubSub), etc.
-
+- [documentation](./_develop/docs)
+- [tools, misc](./_develop)

_develop/docs/README.md

@@ -0,0 +1,52 @@
+README
+======
+
+This directory contains a collection of (hopefully organized and up to date) documentation on 
+running the various backend pieces, development setup, production maintenance, etc. Check out 
+all the markdown documents in this directory tree for information organized by topic.
+
+Development / Setup
+-------------------
+
+- [Most of the general documents for cross-cutting setup of FakeYou.com live here.](./local_development)
+- Some per-application documents (such as for `aichatbot-sidecar`) live as READMEs under that particular 
+  application's code directory, eg. `/crates/{appCategory}/{appName}/README.md` (These are irrelevant 
+  for most folks.)
+
+Web API Overview
+----------------
+
+### FakeYou.com
+
+* [FakeYou.com](https://fakeyou.com) is powered by the following applications:
+    * HTTP APIs:
+        * `storyteller-web` - HTTP API + database monolith for FakeYou
+    * Asynchronous Jobs:
+        * `download-job` - Handle async download of ML models (RVC, SVC, etc.)
+        * `inference-job` - Run async ML model inference (RVC, SVC, SadTalker, etc.) 
+                            This will handle most model inference going forward.
+        * `tts-download-job` - (*deprecated*) Async download of Tacotron2 models only
+        * `tts-inference-job` - (*deprecated*) Async TTS inference of Tacotron2 models only
+        * `w2l-download-job` - (*deprecated*) Async download of videos and images
+        * `w2l-inference-job` - (*deprecated*) Async W2L inference
+
+### PowerStream
+
+* [PowerStream](https://power.stream) (deprecated) is powered by the following applications:
+    * HTTP APIs:
+        * `storyteller-web` - HTTP + database monolith for FakeYou ***and*** PowerStream.
+    * Twitch API integration:
+        * `twitch-pubsub-subscriber` - Dynamically subscribes to Twitch PubSub for "active" users
+    * Websocket events:
+        * `websocket-gateway` - Websocket for users' browsers to pull down enriched Twitch events.
+
+TODO
+----
+
+Notes / TODOs:
+
+* Examples for good Actix+Sqlx Tests:
+  https://stackoverflow.com/questions/65370752/how-do-i-create-an-actix-web-server-that-accepts-both-sqlx-database-pools-and-tr
+
+* Actix/sqlx runtime compat:
+  https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734

_develop/docs/cheat_sheet/migrations.md

@@ -0,0 +1,43 @@
+Migrations guide
+================
+
+## Diesel Migrations
+
+To run pending migrations, execute the following:
+
+```bash
+diesel migration run
+```
+
+To revert the last 3 migrations:
+
+```bash
+diesel migration revert -n 3
+```
+
+To reset the entire database (drop, migrate), run:
+
+```bash
+diesel database reset
+```
+
+## Server Query Codegen
+
+We use SQLx instead of Diesel in the production server. It's typesafe
+SQL instead of an ORM like Diesel.
+
+SQLx connects to a database to derive type information, but obviously
+cannot do this for builds in CI. In order to cache the types, we build
+and check in a cache file (necessary for builds):
+
+```bash
+SQLX_OFFLINE=true cargo sqlx prepare
+```
+
+Now that we have multiple binaries, it's required to include all the queries in the main
+binary so we can generate the cached queries as a single target. That's then executed
+with:
+
+```bash
+SQLX_OFFLINE=true cargo sqlx prepare -- --bin storyteller-web
+```

_develop/docs/local_development/general_setup.md

@@ -0,0 +1,207 @@
+FakeYou Development Setup (bare metal)
+======================================
+
+This document describes how to run the FakeYou infrastructure on your development 
+machine's bare metal (no containers).
+
+Setup instructions and development were tested and known to work on the following
+platforms:
+
+- Mac OS (Apple M2 silicon)
+- Ubuntu 22.04
+
+The applications (binary targets) you might be interested in running include, but 
+are not limited to:
+
+- `storyteller-web`, the HTTP API server
+- `download-job`, which downloads models from the web and stores them in GCS
+- `inference-job`, which downloads models from GCS and runs inference with them
+
+Setup Instructions
+------------------
+
+### (1) Install Rust
+
+Install Rust [using this guide](https://www.rust-lang.org/learn/get-started). If it asks, 
+you'll want "stable" Rust, not "nightly" Rust. If it doesn't ask, it defaults to "stable".
+
+### (2) Install the necessary libraries
+
+Mac: (Lost the list, but it should match Linux. Install Homebrew.)
+
+Ubuntu 22.04:
+
+```bash
+sudo apt install \ 
+  jq \
+  libmysqlclient-dev \
+  libsqlite3-dev \
+  pkgconf
+```
+
+(Note: jq is for combining mysql codegen outputs; pkgconf is if using openssl instead of rustls.)
+
+### (3) Install MySQL server
+
+Mac:
+
+```bash
+brew install mysql
+```
+
+If `mysql -uroot` fails, reboot the machine:
+
+```bash
+sudo reboot now
+```
+
+Ubuntu 22.04:
+
+```bash
+sudo apt install mysql-server
+```
+
+
+### (4) Install a `storyteller` user and table in MySQL
+
+Connect to mysql:
+
+```bash
+# If the following command asks for a password, the password is typically "root"
+sudo mysql -u root -p
+```
+
+Once in MySQL, run the following:
+
+```mysql
+use mysql;
+CREATE DATABASE storyteller;
+CREATE USER 'storyteller'@'localhost' IDENTIFIED BY 'password';
+GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';
+```
+
+Then verify access with `./dev_mysql_connect.sh`
+
+### (5) Install Diesel CLI (for MySQL migrations):
+
+```bash
+cargo install diesel_cli \
+  --no-default-features \
+  --features mysql,sqlite
+```
+
+As of this writing, Mac [has some issues with Diesel CLI](https://github.com/diesel-rs/diesel/issues/2605)
+and requires a few extra dependencies to be installed:
+
+```bash
+brew install libpq
+```
+
+### (6) Run the pending database migrations:
+
+```bash
+diesel migration run
+```
+
+You might get a scary message about `"Encountered unknown type for Mysql: enum"` -- you can ignore this 
+error (see below).
+
+### (7) (optional) Install the SQLx CLI (if doing database development)
+
+```bash
+cargo install sqlx-cli --no-default-features --features rustls,mysql,sqlite
+```
+
+We'll be using diesel to manage the migrations, but sqlx within the app to actually perform queries.
+Diesel is an immature ORM, which isn't a good tech bet, so we use sqlx as at-compile-time
+typesafe SQL.
+
+### (8) Install hosts file:
+
+(This should be the same on Linux and Mac, but might differ for Windows.)
+
+If you're developing against the frontend, it'll target development domains (eg. `dev.fakeyou.com`) instead 
+of `127.0.0.1` or `localhost`. You can make your machine route domains to localhost by editing your hosts 
+file (located at `/etc/hosts`) to include the following configuration lines:
+
+```
+127.0.0.1    dev.fakeyou.com
+127.0.0.1    api.dev.fakeyou.com
+127.0.0.1    devproxy.fakeyou.com
+
+127.0.0.1    dev.storyteller.ai
+127.0.0.1    api.dev.storyteller.ai
+127.0.0.1    devproxy.storyteller.ai
+```
+
+### (9a) Install Redis
+
+```
+sudo apt install redis
+```
+
+### (9b) Run one or more applications:
+DO NOT CHECK THESE TWO FILES INTO THE REPO
+Ask Brandon for the .env-secrets and .storteller-web.development-secrets.env and place it in the root of the folder.
+
+To start the HTTP API server,
+
+```bash
+cargo run --bin storyteller-web
+```
+Note that this compiles and runs the "development" binary. It's faster and easier to debug than the
+optimized "production" build. To build a fully optimized production release,
+run `cargo build --release --bin storyteller-web` . Note that this will take much longer.
+
+DO NOT CHECK THESE TWO FILES INTO THE REPO
+
+Ask Brandon for the .download-job.development-secrets.env and place it in
+crates/service/job/download_job/config
+
+Ask Brandon for the .inference-job.development-secrets.env and place it in
+crates/service/job/inference_job/config
+
+To download some ML models, run:
+
+```bash
+cargo run --bin download-job
+cargo run --bin tts-download-job
+```
+
+If you want to run both the HTTP API and the jobs, you'll need to run both processes in different
+terminals or tmux sessions.
+
+Note also that the configurations pointing to the ML monorepo must be set up for each application.
+Additionally, some development secrets may be needed (ask the team to share).
+
+To execute ML inference, run:
+
+```bash
+cargo run --bin inference-job
+cargo run --bin tts-inference-job
+```
+
+Again, note that the configurations pointing to the ML monorepo must be set up for each application.
+Additionally, some development secrets may be needed (ask the team to share).
+
+Solutions to Common Setup Problems 
+-----------------------------------
+
+### [Fix] Sqlx Error when Performing Schema Migrations
+
+You might get this error message during migration,
+
+```
+Encountered unknown type for Mysql: enum
+thread 'main' panicked at 'internal error: entered unreachable code: Mysql only supports a closed set of types.
+                         If you ever see this error message please open an issue at https://github.com/diesel-rs/diesel 
+                         containing a dump of your schema definition.', diesel_cli/src/print_schema.rs:310:17
+note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
+```
+
+I haven't found the cause (it doesn't happen on newer installs), but the migrations appear to work regardless 
+of this error message. You can essentially ignore it.
+
+### [Fix] Can't connect to local MySQL after install on Ubuntu
+
+- If MySql in local dev can't be connected to, [reset the accounts](https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux).

_develop/docs/local_development/local_development_k8s.md

@@ -0,0 +1,22 @@
+# Local Kubernetes environment for standing up the APIs and their dependencies
+
+(These are Paul's notes.)
+
+- Install minikube, kubectl, docker, and musl from your OS package manager
+- `rustup target add x86_64-unknown-linux-musl`
+- `cargo build --release --bin storyteller-web --target=x86_64-unknown-linux-musl`
+- `minikube start --extra-config=apiserver.service-node-port-range=1-65535`
+- `eval $(minikube docker-env)`
+- `minikube mount /path/to/storyteller-rust/:/storyteller-rust &> /dev/null &`
+- `docker build -t storyteller-web -f localdev/Dockerfile .`
+- `kubectl apply -f localdev/kubernetes.yml`
+- Get the ip address of minikube with `minikube ip`, this is where all the services will be running on their 
+regular ports (12345 for the api, 6379 for redis, 3306 for mysql)
+- That's it, we just spun up mysql, applied database migrations, spun up redis, and spun up our api
+
+# Development tips
+- Rebuild and "deploy" (locally) with `cargo build --release --bin storyteller-web --target=x86_64-unknown-linux-musl && kubectl rollout restart storyteller-web`
+- View logs from the api with `kubectl logs  storyteller-web<TAB>`
+- Delete everything minikube (if you start running out of disk space or what-have-you) with `minikube delete`
+- See pod status with `kubectl get pods`
+- If there are errors you can get more info with `kubectl describe pods`

_develop/docs/local_development/local_development_nginx_proxy.md

@@ -0,0 +1,56 @@
+Local Development nginx proxy
+=============================
+
+It isn't necessary to use nginx for development, but it can be nice. Nginx allows for a more 
+sophisticated testing setup that makes cookie and CORS issues less of a pain.
+
+You can use nginx to serve (locally-signed) SSL, redirect traffic to different ports, or run 
+multiple copies of the app side by side.
+
+### Hosts file
+
+Set the following configs in your `/etc/hosts` file (as you would probably do even without nginx),
+
+```
+127.0.0.1    dev.fakeyou.com
+127.0.0.1    api.dev.fakeyou.com
+127.0.0.1    devproxy.fakeyou.com
+
+127.0.0.1    dev.storyteller.ai
+127.0.0.1    api.dev.storyteller.ai
+127.0.0.1    devproxy.storyteller.ai
+
+# Deprecated development hostnames :
+127.0.0.1    jungle.horse
+127.0.0.1    api.jungle.horse
+```
+
+### Nginx installation and administration
+
+Configure Nginx per the checked in Nginx configs (and instructions) in
+the directory `/_develop/localdev/nginx-http-config`.
+
+This should be straightforward, but if it isn't, contribute docs here.
+
+### [Fix] Python 3.6 on Apple M1 Mac
+
+Python3.6 isn't supported on Apple silicon, and it's not in homebrew. It can be installed with 
+[nix using Rosetta](https://stackoverflow.com/a/65980989):
+
+Download: https://nixos.org/download.html#nix-quick-install
+
+```
+nix run nixpkgs.python36 -c python
+```
+
+Install venv:
+
+```
+nix run nixpkgs.python36 -c python -m venv python
+```
+
+Install other packages on Mac that aren't used in venv:
+
+```
+python3 -m pip install --user requests gdown youtube_dl
+```

_develop/docs/production/production_database.md

@@ -0,0 +1,17 @@
+Production Database
+===================
+
+Secrets can be subbed in for `.env` and the standard diesel migration tooling will work.
+
+Dump a MySQL report
+-------------------
+
+```bash
+mysql -u storyteller \
+  -pPasswordHere \
+  -h IP_HERE \
+  -P 3306 \
+  -D storyteller \
+  -B \
+  -e "select * from voice_clone_requests;" | sed "s/'/\'/;s/\t/\",\"/g;s/^/\"/;s/$/\"/;s/\n//g" > voice_clone_report.csv
+```

_develop/docs/schema/database_schema.md

@@ -1,6 +1,8 @@
 database schema
 ================
 
+(This is woefully incomplete and unmaintained.)
+
 Twitch Model
 ------------
 

_develop/localdev/Dockerfile

@@ -0,0 +1,23 @@
+FROM rust:latest
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -y \
+        mariadb-client \
+        imagemagick  \
+        libmariadb-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+
+RUN cargo install diesel_cli --features mysql && \
+  cargo install sqlx-cli --features mysql
+
+WORKDIR /storyteller-rust
+
+ARG GIT_SHA
+
+RUN echo -n ${GIT_SHA} > /GIT_SHA
+
+CMD ["/storyteller-rust/localdev/startup.sh"]
+

_develop/localdev/kubernetes.yml

@@ -0,0 +1,153 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata: 
+  name: mysql
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mysql
+  template:
+    metadata:
+      labels:
+        app: mysql
+    spec:
+      containers:
+      - name: mysql
+        image: mysql:8.0.30
+        env:
+        - name: MYSQL_USER
+          value: storyteller
+        - name: MYSQL_PASSWORD
+          value: password
+        - name: MYSQL_DATABASE
+          value: storyteller
+        - name: MYSQL_ROOT_PASSWORD
+          value: password
+        ports:
+          - containerPort: 3306 
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata: 
+  name: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+      - name: redis
+        image: redis:7.0.4
+        ports:
+          - containerPort: 6379
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata: 
+  name: storyteller-web
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: storyteller-web
+  template:
+    metadata:
+      labels:
+        app: storyteller-web
+    spec:
+      containers:
+      - name: storyteller-web
+        image: storyteller-web:latest
+        imagePullPolicy: Never
+        env:
+          - name: MYSQL_URL
+            value: mysql://storyteller:password@mysql.default/storyteller
+          - name: REDIS_0_URL
+            value: redis://redisd@redis.default/0
+          - name: REDIS_1_URL
+            value: redis://redisd@redis.default/1
+          - name: ACCESS_KEY
+            value: garbage
+          - name: SECRET_KEY
+            value: garbage
+          - name: REGION_NAME
+            value: garbage
+          - name: W2L_PRIVATE_DOWNLOAD_BUCKET_NAME
+            value: garbage
+          - name: W2L_PUBLIC_DOWNLOAD_BUCKET_NAME
+            value: garbage
+          - name: AUDIO_UPLOADS_BUCKET_ROOT
+            value: garbage
+          - name: TWITCH_USERNAME
+            value: garbage
+          - name: TWITCH_STREAM_KEY
+            value: garbage
+          - name: TWITCH_APP_CLIENT_ID
+            value: garbage
+          - name: TWITCH_APP_CLIENT_SECRET
+            value: garbage
+        ports:
+          - containerPort: 12345
+        volumeMounts:
+          - mountPath: /storyteller-rust
+            name: storyteller-rust-folder
+      volumes:
+        - name: storyteller-rust-folder
+          hostPath:
+            path: /storyteller-rust
+---
+
+apiVersion: v1
+kind: Service
+metadata: 
+  name: mysql
+spec:
+  type: NodePort
+  ports:
+    - name: mysql
+      port: 3306
+      nodePort: 3306
+  selector:
+    app: mysql
+
+---
+
+apiVersion: v1
+kind: Service
+metadata: 
+  name: redis
+spec:
+  type: NodePort
+  ports:
+    - name: redis
+      port: 6379
+      nodePort: 6379
+  selector:
+    app: redis
+
+---
+
+apiVersion: v1
+kind: Service
+metadata: 
+  name: storyteller-web
+spec:
+  type: NodePort
+  ports:
+    - name: storyteller-web
+      port: 12345
+      nodePort: 12345
+  selector:
+    app: storyteller-web
+
+

_develop/localdev/nginx-http-config/storyteller-ssl

@@ -0,0 +1,258 @@
+# Nginx configs for "jungle.horse" and "api.jungle.horse" (http)
+#
+# On Ubuntu Linux, this config is located at the following location:
+# 
+#  - /etc/nginx/sites-enabled/{name-of-site}
+#
+# Move this file there, and then restart with,
+#
+#   sudo service nginx restart
+#
+# On Mac, this is located at the following locations:
+#
+#  - /opt/homebrew/etc/nginx/servers/{name-of-site} (for nginx installed with homebrew)
+#  - /usr/local/etc/nginx/sites-enabled/{name-of-site} (for default nginx install)
+#
+#
+# On Mac, verify configs and restart with,
+#
+#   sudo nginx -t
+#   sudo nginx -s reload
+#
+#           ================
+#           SSL Instructions
+#           ================
+#
+# You can launch Chrome in a way it disregards self-signed certs and CORS:
+#
+#     chromium  --disable-web-security --ignore-certificate-errors --user-data-dir=~/chromeTemp
+#
+# To create self-signed certs on Ubuntu Linux,
+#
+#     sudo apt-get install openssl
+#
+#     sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/localhost.key -out /etc/ssl/certs/localhost.crt
+#
+#     ^ most fields can be blank, but the "Common Name" should be `*.jungle.horse`
+#
+#     sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/devfakeyou.key -out /etc/ssl/certs/devfakeyou.crt
+#     sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/devstoryteller.key -out /etc/ssl/certs/devstoryteller.crt
+#
+#     ^ most fields can be blank, but the "Common Name" should be `*.dev.fakeyou.com` and `*.dev.storyteller.io` respectively.
+#
+# For more: https://www.cloudsavvyit.com/1306/how-to-create-and-use-self-signed-ssl-on-nginx/
+#
+# To create self-signed certs on Mac,
+#
+#     openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt
+#
+#     ^ most fields can be blank, but the "Common Name" should be `*.jungle.horse`
+#
+# On Mac, load the self signed cert into the keychain with the following command:
+#
+#     sudo security add-trusted-cert \
+#      -d -r trustRoot \
+#      -k /Library/Keychains/System.keychain /etc/ssl/certs/localhost.crt
+#
+# Self signed certs might be considered insecure by the browser. In Firefox, 
+# set about:config `security.tls.insecure_fallback_hosts` to include a list of
+# permitted domains. You may also need to configure other settings. 
+#
+#
+
+# ========================================================
+#                      FAKEYOU
+# ========================================================
+
+# FakeYou javascript frontend over plain HTTP (launched by Yarn)
+server {
+    listen 80;
+    server_name dev.fakeyou.com;
+    location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:7000;
+        proxy_redirect off;
+    }
+}
+
+# FakeYou API backend over plain HTTP (Rust app)
+server {
+    listen 80;
+    server_name api.dev.fakeyou.com;
+    location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:12345;
+        proxy_redirect off;
+    }
+}
+
+# FakeYou javascript frontend over SSL (launched by Yarn)
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    ssl_certificate /etc/ssl/certs/devfakeyou.crt;
+    ssl_certificate_key /etc/ssl/private/devfakeyou.key;
+    ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
+    server_name dev.fakeyou.com;
+    location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:7000;
+        proxy_redirect off;
+    }
+}
+
+# FakeYou API backend over SSL (Rust app)
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    ssl_certificate /etc/ssl/certs/devfakeyou.crt;
+    ssl_certificate_key /etc/ssl/private/devfakeyou.key;
+    ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
+    server_name api.dev.fakeyou.com;
+    location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:12345;
+        proxy_redirect off;
+    }
+}
+
+# ========================================================
+#                      STORYTELLER
+# ========================================================
+
+# Storyteller javascript frontend over plain HTTP (launched by Yarn)
+server {
+    listen 80;
+    server_name dev.storyteller.io;
+    location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:7000;
+        proxy_redirect off;
+    }
+}
+
+# Storyteller API backend over plain HTTP
+server {
+    listen 80;
+    server_name api.dev.storyteller.io;
+    location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:12345;
+        proxy_redirect off;
+    }
+}
+
+# Storyteller javascript frontend over SSL (launched by Yarn)
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    ssl_certificate /etc/ssl/certs/devstoryteller.crt;
+    ssl_certificate_key /etc/ssl/private/devstoryteller.key;
+    ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
+    server_name dev.storyteller.io;
+    location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:7000;
+        proxy_redirect off;
+    }
+}
+
+# Storyteller API backend over SSL
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    ssl_certificate /etc/ssl/certs/devstoryteller.crt;
+    ssl_certificate_key /etc/ssl/private/devstoryteller.key;
+    ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
+    server_name api.dev.storyteller.io;
+    location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:12345;
+        proxy_redirect off;
+    }
+}
+
+# ========================================================
+#            OLD PRODUCTS THAT NEED UPDATING
+# ========================================================
+
+#     ## TODO(2022-02-13): This is new and untested
+#     # WebSocket API backend over plain HTTP
+#     server {
+#         listen 80;
+#         server_name ws.jungle.horse;
+#         location / {
+#             proxy_set_header Host $host;
+#             proxy_pass http://127.0.0.1:54321;
+#             proxy_redirect off;
+#         }
+#     }
+#
+#     # Websocket API backend over SSL
+#     server {
+#         listen 443 ssl;
+#         listen [::]:443 ssl;
+#         ssl_certificate /etc/ssl/certs/localhost.crt;
+#         ssl_certificate_key /etc/ssl/private/localhost.key;
+#         ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
+#         server_name ws.jungle.horse;
+#         location / {
+#             proxy_set_header Host $host;
+#             proxy_pass http://127.0.0.1:54321;
+#             proxy_redirect off;
+#         }
+#     }
+#
+#     # Javascript frontend over plain HTTP
+#     server {
+#         listen 80;
+#         server_name jungle.horse;
+#         location / {
+#             proxy_set_header Host $host;
+#             proxy_pass http://127.0.0.1:7000;
+#             proxy_redirect off;
+#         }
+#     }
+#
+#     # TODO(2022-02-13): This is new and untested
+#     # Javascript frontend (OBS Layer) over plain HTTP
+#     server {
+#         listen 80;
+#         server_name obs.jungle.horse;
+#         location / {
+#             proxy_set_header Host $host;
+#             proxy_pass http://127.0.0.1:7007;
+#             proxy_redirect off;
+#         }
+#     }
+#
+#     # Javascript frontend over SSL
+#     server {
+#         listen 443 ssl;
+#         listen [::]:443 ssl;
+#         ssl_certificate /etc/ssl/certs/localhost.crt;
+#         ssl_certificate_key /etc/ssl/private/localhost.key;
+#         ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
+#         server_name jungle.horse;
+#         location / {
+#             proxy_set_header Host $host;
+#             proxy_pass http://127.0.0.1:7000;
+#             proxy_redirect off;
+#         }
+#     }
+#
+#     # Javascript frontend over SSL (OBS Layer)
+#     server {
+#         listen 443 ssl;
+#         listen [::]:443 ssl;
+#         ssl_certificate /etc/ssl/certs/localhost.crt;
+#         ssl_certificate_key /etc/ssl/private/localhost.key;
+#         ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
+#         server_name obs.jungle.horse;
+#         location / {
+#             proxy_set_header Host $host;
+#             proxy_pass http://127.0.0.1:7007;
+#             proxy_redirect off;
+#         }
+#     }
+#

_develop/localdev/startup.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set +e +o pipefail
+diesel migration run --database-url="${MYSQL_URL}"
+set -e -o pipefail
+./target/x86_64-unknown-linux-musl/release/storyteller-web
+

_develop/localdev/ubuntu_setup.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+echo 'Installing APT packages...'
+
+# OpenSSL requirements: libssl-dev, pkg-config
+# Database requirements: mysql-server, libmysqlclient-dev
+# Other tools: jq (db codegen)
+sudo apt-get install -y \
+  jq \
+  libmysqlclient-dev \
+  libssl-dev \
+  mysql-server \
+  pkg-config \
+  --no-install-recommends
+
+echo 'Installing Cargo packages...'
+
+# Install Diesel CLI for migrations
+cargo install diesel_cli \
+  --no-default-features \
+  --features mysql
+
+# Install SQLx for codegen
+cargo install sqlx-cli \
+  --no-default-features \
+  --features rustls,mysql
+
+echo 'Installing Database user...'
+
+# Create DB account (using default system mysql password)
+sudo mysql \
+  -u "root" \
+  -p"password" \
+  -e "use mysql; CREATE DATABASE IF NOT EXISTS storyteller; CREATE USER IF NOT EXISTS 'storyteller'@'localhost' IDENTIFIED BY 'password'; GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';"
+
+

_develop/tools/.gitignore

@@ -0,0 +1,6 @@
+# Secrets
+db_secrets.toml
+
+# Python Virtual Env
+python/
+

_develop/tools/find_101soundboards_accounts.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+
+# FIXME: Needs packages 'toml' and 'mysql-connector-python'. Put in requirements.txt.
+
+import time
+import toml
+import mysql.connector
+
+DB_SECRETS_FILE = 'db_secrets.toml'
+
+# TODO: Share db-related setup when more scripts are added.
+
+def read_database_secrets(secrets_file = DB_SECRETS_FILE):
+    secrets = None
+    with open(secrets_file, "r") as f:
+        secrets = toml.load(f)
+
+    required_keys = ['database', 'hostname', 'port', 'password', 'username']
+    for key in required_keys:
+        if key not in secrets:
+            raise Exception(f"Required key '#{key}' not present")
+
+    return secrets
+
+def connect_to_mysql():
+    secrets = read_database_secrets()
+    connection = mysql.connector.connect(
+        host = secrets['hostname'],
+        port = secrets['port'],
+        user = secrets['username'],
+        password = secrets['password'],
+        database = secrets['database'],
+    )
+    return connection
+
+connection = connect_to_mysql()
+cursor = connection.cursor()
+
+SAFE_ID = 76000000 # Reasonably completed on 2022-06-24
+LIMIT = 100000
+
+#def get_record_count():
+#    query = f"SELECT COUNT(*) FROM tts_inference_jobs"
+#    cursor.execute(query)
+#    result = cursor.fetchall()
+#    print(result)
+#    return result[0]
+#
+##total_count = get_record_count()
+
+#total_count = 75082057
+total_count = 21925602
+
+def get_user_tokens(cursor):
+    query = f"""
+		SELECT
+			r.maybe_creator_user_token, r.creator_ip_address
+		FROM
+			tts_results as r
+				LEFT OUTER JOIN
+			users as u
+			ON
+					r.maybe_creator_user_token = u.token
+		WHERE
+				r.raw_inference_text LIKE '%spidermant%'
+		AND r.created_at > ( CURDATE() - INTERVAL 1 HOUR )
+    """
+    cursor.execute(query)
+    results = cursor.fetchall()
+
+    user_tokens = [row[0] for row in results]
+    user_tokens = [token for token in user_tokens if token is not None]
+    user_tokens = set(user_tokens)
+
+    ip_addresses = [row[1] for row in results]
+    ip_addresses = [ip for ip in ip_addresses if ip is not None]
+    ip_addresses = set(ip_addresses)
+
+    return (user_tokens, ip_addresses)
+
+(user_tokens, ip_addresses) = get_user_tokens(cursor)
+
+print(user_tokens)
+print(len(user_tokens))
+print(len(ip_addresses))
+
+with open('101_user_tokens.txt', 'w') as f:
+    for user_token in user_tokens:
+        f.write(user_token)
+        f.write("\n")
+
+with open('101_ip_addresses.txt', 'w') as f:
+    for ip in ip_addresses:
+        f.write(ip)
+        f.write("\n")
+
+
+
+
+

_develop/tools/requirements.txt

@@ -0,0 +1,3 @@
+mysql-connector-python==8.0.32
+protobuf==3.20.3
+toml==0.10.2

_docs/local_development_setup.md

@@ -1,207 +0,0 @@
-dev setup
-=========
-
-[Back to main README](../README.md)
-
-### Debugging and Fixing CUDA/PyTorch
-
-See this spreadsheet for previous battles with version incompatibility:
-https://docs.google.com/spreadsheets/d/1BEdLmwOzo3r83-iJn9sj6co1VT92t3lIeJ-lNH25bdQ/edit#gid=0
-
-Ubuntu might upgrade the driver by accident, and it might need reinstallation. Who knows.
-This stuff is a nightmare.
-
-### Database Setup
-
-Install the following libraries, and see the notes further below about MySQL on Ubuntu 20.04.
-
-```
-mysql-server
-imagemagick
-```
-
-To manage the database and perform migrations, install the Rust tools diesel and sqlx.
-
-We'll be using diesel to manage the migrations, but sqlx within the app to actually perform queries.
-Diesel is an ORM, which is dumb, so we use sqlx as at-compile-time typesafe SQL.
-
-```
-sudo apt-get install libmysqlclient-dev
-cargo install sqlx-cli --no-default-features --features mysql [2022-01-16: is this needed?]
-```
-
-To install a more modern version of Diesel (unreleased) that 
-supports the "migrations_dir" toml config option:
-
-```
-cargo install diesel_cli \
-  --git https://github.com/diesel-rs/diesel.git \
-  --rev a213fe232a122f35a812b0ce0269708a1845a4c9 \
-  --no-default-features \
-  --features mysql
-```
-
-#### Sqlx Error on Linux when Performing Schema Migrations
-
-You might get this error message during migration,
-
-```
-Encountered unknown type for Mysql: enum
-thread 'main' panicked at 'internal error: entered unreachable code: Mysql only supports a closed set of types.
-                         If you ever see this error message please open an issue at https://github.com/diesel-rs/diesel containing a dump of your schema definition.', diesel_cli/src/print_schema.rs:310:17
-note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
-```
-
-I haven't found the cause (it doesn't happen on Mac), but the migrations appear to work regardless of the error message.
-
-#### Linux database notes
-
-If MySql in local dev can't be connected to, reset the accounts:
-
-https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux
-
-#### Fixing dev MySql on Ubuntu 20.04
-
-For some reason, the MySql default install on 20.04 gave me a bunch of trouble.
-
-In retrospect, I _think_ this is because 'root@localhost' requires sudo to access, but if this
-gives any trouble in the future, here's how I got around it (two full hours of distraction!)
-
-```
-sudo apt-get install mysql-server
-
-# But for some reason the default password doesn't work and diesel/sqlx can't connect?
-
-# Kill everything
-
-sudo apt purge mysql-server mysql-client mysql-common
-sudo apt autoremove
-sudo mv -iv /var/lib/mysql /var/tmp/mysql-backup
-sudo rm -rf /var/lib/mysql*
-
-sudo /usr/bin/mysql_secure_installation
-
-# still no work...
-
-sudo systemctl stop mysql.service
-
-# ugh, it wasn't chowned or created (both states observed in different installs)…
-sudo mkdir -p /var/run/mysqld
-sudo chown mysql:mysql /var/run/mysqld
-
-sudo mysqld_safe --skip-grant-tables --skip-networking &
-
-mysql -u root
-
-flush privileges;
-USE mysql;
-ALTER USER 'root'@'localhost' IDENTIFIED BY 'root';
-
-sudo killall -u mysql
-sudo systemctl restart mysql.service
-
-# now it works
-sudo mysql -u root -p
-
-# but now I have to use "sudo" !?!?
-
-use mysql;
-
-CREATE USER 'storyteller'@'localhost' IDENTIFIED BY 'password';
-GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';
-```
-
-### Install Diesel (migrations only)
-
-We use Diesel to manage migrations, but we don't use it in server code.
-Actual server code uses SQLx. To install the CLI tool, run the following:
-
-```
-cargo install diesel_cli
-```
-
-Mac [has issues](https://github.com/diesel-rs/diesel/issues/2605) and requires a few dependencies:
-
-```
-brew install libpq
-```
-
-### Database migrations
-
-To reset the entire database (drop, migrate), run:
-
-```
-diesel database reset
-```
-
-To migrate at the current step and beyond:
-
-```
-diesel migration run
-```
-
-To undo migrations at the current step:
-
-```
-diesel migration redo
-```
-
-### Server Query Codegen
-
-We use SQLx instead of Diesel in the production server. It's typesafe
-SQL instead of an ORM like Diesel.
-
-SQLx connects to a database to derive type information, but obviously
-cannot do this for builds in CI. In order to cache the types, we build
-and check in a cache file (necessary for builds):
-
-```
-SQLX_OFFLINE=true cargo sqlx prepare
-```
-
-Now that we have multiple binaries, it's required to include all the queries in the main
-binary so we can generate the cached queries as a single target. That's then executed
-with:
-
-```
-SQLX_OFFLINE=true cargo sqlx prepare -- --bin storyteller-web
-```
-
-### Setting Up a Local Nginx Proxy
-
-Set up a local nginx to proxy to the frontend and backend so cookie issues aren't annoying
-
-Configure Nginx per the checked in Nginx configs (and instructions) in `localdev/nginx-http-config`.
-
-```
-
-And in /etc/hosts,
-
-```
-127.0.0.1  jungle.horse
-127.0.0.1  api.jungle.horse
-
-```
-
-### Python 3.6 on Apple M1 Mac
-
-Python3.6 isn't supported on Apple silicon, and it's not in homebrew. It can be installed with 
-[nix using Rosetta](https://stackoverflow.com/a/65980989):
-
-Download: https://nixos.org/download.html#nix-quick-install
-
-```
-nix run nixpkgs.python36 -c python
-```
-
-Install venv:
-
-```
-nix run nixpkgs.python36 -c python -m venv python
-```
-
-Install other packages on Mac that aren't used in venv:
-
-```
-python3 -m pip install --user requests gdown youtube_dl
-```

_migrations/2021-05-15-071534_users/up.sql

@@ -1,235 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-CREATE TABLE users (
-  -- Not used for anything except replication.
-  id BIGINT(20) NOT NULL AUTO_INCREMENT,
-
-  -- Visible "primary key"
-  token VARCHAR(32) NOT NULL,
-
-  -- Username is a lookup key; display_name allows the user to add custom case.
-  username VARCHAR(20) NOT NULL,
-  display_name VARCHAR(20) NOT NULL,
-
-  email_address VARCHAR(255) NOT NULL,
-  email_confirmed BOOLEAN NOT NULL DEFAULT false,
-
-  -- The role assigned to the user confers permissions.
-  user_role_slug VARCHAR(16) NOT NULL,
-
-  -- ========== CREDENTIALS ==========
-
-  -- Bcrypt password hash. Granted, there are newer methods:
-  -- https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html
-  -- Passwords may be a max of 64 characters per BCrypt.
-  password_hash BINARY(60) NOT NULL,
-
-  -- Incremented with every update to the password.
-  password_version INT NOT NULL DEFAULT 0,
-
-  -- ========== ABUSE TRACKING ==========
-
-  -- For abuse tracking.
-  -- Wide enough for IPv4/6
-  ip_address_creation VARCHAR(40) NOT NULL,
-  ip_address_last_login VARCHAR(40) NOT NULL,
-  ip_address_last_update VARCHAR(40) NOT NULL,
-
-  -- ========== DISPLAY / PROFILE ==========
-
-  -- Gravatar image hashes are precomputed.
-  email_gravatar_hash CHAR(32) NOT NULL,
-
-  -- The profile of the user in markdown (user editable).
-  profile_markdown TEXT NOT NULL,
-
-  -- Generated HTML (not user-editable).
-  profile_rendered_html TEXT NOT NULL,
-
-  -- An uploaded avatar. Public hash in our bucket.
-  -- avatar_public_bucket_hash CHAR(32) DEFAULT NULL,
-
-  -- ========== SOCIAL MEDIA ==========
-
-  -- Social media usernames
-  -- These are not confirmed. We'll need to build an OAuth system to handle that.
-  discord_username VARCHAR(36) DEFAULT NULL,
-  twitter_username VARCHAR(36) DEFAULT NULL,
-  twitch_username VARCHAR(36) DEFAULT NULL,
-  patreon_username VARCHAR(36) DEFAULT NULL,
-  github_username VARCHAR(36) DEFAULT NULL,
-  cashapp_username VARCHAR(36) DEFAULT NULL,
-  website_url VARCHAR(255) DEFAULT NULL,
-
-  -- ========== USER PREFERENCES ==========
-
-  -- If the user doesn't want to use gravatar and doesn't have an uploaded avatar.
-  disable_gravatar BOOLEAN NOT NULL DEFAULT false,
-
-  -- (THIS MIGHT NOT BE USED)
-  -- NB: DO NOT SORT!
-  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
-  -- Visibility preference used at TTS inference time (which can be changed on the result itself).
-  -- Moderators will still see them.
-  preferred_tts_result_visibility ENUM(
-    'public',
-    'hidden',
-    'private'
-  ) NOT NULL DEFAULT 'public',
-
-  -- (THIS MIGHT NOT BE USED)
-  -- NB: DO NOT SORT!
-  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
-  -- Visibility preference used at TTS inference time (which can be changed on the result itself).
-  -- Moderators will still see them.
-  preferred_w2l_result_visibility ENUM(
-    'public',
-    'hidden',
-    'private'
-  ) NOT NULL DEFAULT 'public',
-
-  -- Auto play preferences
-  -- Tri-state because we want to potentially change the default behavior.
-  auto_play_audio_preference BOOLEAN DEFAULT NULL,
-  auto_play_video_preference BOOLEAN DEFAULT NULL,
-
-  -- Favorite TTS voice to use by default
-  -- maybe_preferred_tts_model_token VARCHAR(32) DEFAULT NULL,
-
-  -- Favorite W2L model to use by default
-  -- maybe_preferred_w2l_template_token VARCHAR(32) DEFAULT NULL,
-
-  -- Settings
-  -- DO NOT REORDER.
-  -- dark_mode_preference ENUM(
-  --   'light-mode',
-  --   'dark-mode',
-  --   'use-clock'
-  --  ) NOT NULL DEFAULT 'light-mode',
-
-  -- ========== STATS ==========
-
-  -- For tracking stats.
-  -- The "cached" values are updated by a background job.
-  -- cached_tts_rendered_counter INT(10) NOT NULL DEFAULT 0,
-  -- cached_w2l_rendered_counter INT(10) NOT NULL DEFAULT 0,
-
-  -- ========== MODERATION DETAILS ==========
-
-  -- Different than deleted.
-  -- Users still show up, but can't do anything.
-  is_banned BOOLEAN NOT NULL DEFAULT false,
-
-  -- If a moderator has comments.
-  maybe_mod_comments VARCHAR(255) DEFAULT NULL,
-
-  -- The last moderator that made changes.
-  maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
-
-  -- ========== VECTOR CLOCK ==========
-
-  -- Incremented with every update.
-  version INT NOT NULL DEFAULT 0,
-
-  -- ========== TIMESTAMPS ==========
-
-  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-
-  -- If the user is deleted, we set this.
-  -- This is different than banned. These users won't show up at all.
-  user_deleted_at TIMESTAMP NULL,
-  mod_deleted_at TIMESTAMP NULL,
-
-  -- INDICES --
-  PRIMARY KEY (id),
-  UNIQUE KEY (token),
-  UNIQUE KEY (username),
-  UNIQUE KEY (email_address),
-  KEY fk_user_role_slug (user_role_slug)
-
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
-
-CREATE TABLE user_roles (
-  -- Not used for anything except replication.
-  id BIGINT(20) NOT NULL AUTO_INCREMENT,
-
-  -- Effective "primary key"
-  slug CHAR(16) NOT NULL,
-
-  name VARCHAR(255) NOT NULL,
-
-  -- Usage
-  can_use_tts BOOLEAN NOT NULL DEFAULT FALSE,
-  can_use_w2l BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_own_tts_results BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_own_w2l_results BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_own_account BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- Contribution
-  can_upload_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
-  can_upload_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_own_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_own_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- Moderation
-  can_approve_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
-  -- Edit includes "lock_from_use" and "lock_from_modification"
-  can_edit_other_users_profiles BOOLEAN NOT NULL DEFAULT FALSE,
-  can_edit_other_users_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
-  can_edit_other_users_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_other_users_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_other_users_tts_results BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_other_users_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_other_users_w2l_results BOOLEAN NOT NULL DEFAULT FALSE,
-  can_ban_users BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_users BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- Incremented with every update.
-  version INT NOT NULL DEFAULT 0,
-
-  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-
-  -- INDICES --
-  PRIMARY KEY (id),
-  UNIQUE KEY (slug)
-
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
-
-CREATE TABLE user_sessions (
-  -- Not used for anything except replication.
-  id BIGINT(20) NOT NULL AUTO_INCREMENT,
-
-  -- Session entropy
-  token VARCHAR(32) NOT NULL,
-
-  -- Foreign key to user
-  user_token VARCHAR(32) NOT NULL,
-
-  -- Track each session's creation IP.
-  -- Wide enough for IPv4/6
-  ip_address_creation VARCHAR(40) NOT NULL,
-
-  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-
-  -- Session termination time.
-  -- This must be set by the server code, or the session is invalid
-  expires_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-
-  -- deletion = session termination
-  -- Typically these are destroyed by users, but if in the future we allow mods to
-  -- delete them, it doesn't really matter who did the deletion: sessions are not
-  -- designed to be recoverable.
-  deleted_at TIMESTAMP NULL,
-
-  -- INDICES --
-  PRIMARY KEY (id),
-  UNIQUE KEY (token),
-  KEY fk_user_token (user_token),
-  KEY index_ip_address_creation (ip_address_creation)
-
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_migrations/2022-08-05-202017_generic_download_jobs/up.sql

@@ -1,110 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-CREATE TABLE generic_download_jobs (
-  -- Not used for anything except replication.
-  id BIGINT(20) NOT NULL AUTO_INCREMENT,
-
-  -- Effective "primary key" (PUBLIC)
-  -- This is so the in-progress results can be looked up by the UI.
-  token VARCHAR(32) NOT NULL,
-
-  -- Idempotency token
-  -- This is so the frontend client doesn't submit duplicate jobs.
-  uuid_idempotency_token VARCHAR(36) NOT NULL,
-
-  -- ========== SUCCESS CASE ==========
-
-  -- If the job completes successfully, this is the result token.
-  -- This is only populated on a successful result.
-  on_success_downloaded_entity_token VARCHAR(32) DEFAULT NULL,
-
-  -- The type of the object will vary based on the type of the upload,
-  -- and we may include heuristics that auto-detect types in the future
-  on_success_downloaded_entity_type VARCHAR(32) DEFAULT NULL,
-
-  -- ========== UPLOAD DETAILS ==========
-
-  -- A "well defined" type of upload
-  -- The first upload type will be "hifigan_model"
-  download_type VARCHAR(32) NOT NULL,
-
-  -- Assume the same URL can be uploaded more than once (in case of errors or
-  -- people sharing models)
-
-  download_url VARCHAR(512) NOT NULL,
-
-  -- The user-supplied title of the upload
-  -- This might have different meanings for different upload types
-  -- The "name" of the voice model, which might be complicated.
-  -- If maybe_subject_token (etc.) is set, then it's authoritative instead.
-  title VARCHAR(255) NOT NULL,
-
-  -- ========== CREATOR DETAILS AND PREFERENCES ==========
-
-  -- Foreign key to user
-  creator_user_token VARCHAR(32) NOT NULL,
-
-  -- For abuse tracking.
-  -- Wide enough for IPv4/6
-  creator_ip_address VARCHAR(40) NOT NULL,
-
-  -- The creator can set a desired visibility for their data.
-  -- This does not always apply to every upload type.
-  -- Additionally, some upload types may require moderator approval prior
-  -- to being publicly listed, and this field has no bearing on that.
-  -- NB: DO NOT CHANGE THE ORDER OF THE ENUM VALUES DURING SCHEMA MIGRATIONS.
-  creator_set_visibility ENUM(
-    'public',
-    'hidden',
-    'private'
-  ) NOT NULL DEFAULT 'public',
-
-  -- ========== JOB SYSTEM DETAILS ==========
-
-  -- Jobs begin as "pending", then transition to other states.
-  --
-  --  * Pending = job is ready to go
-  --  * Started = job is running
-  --  * Complete_Success = job is done (success)
-  --  * Complete_Failure = job is done (failure)
-  --  * Attempt_Failed = job failed but may retry.
-  --  * Dead = job failed permanently.
-  --
-  -- Pending -> Started -> Complete_Success
-  --                    |-> Complete_Failure
-  --                    \-> Attempt_Failed -> Started -> { Complete, Failed, Dead }
-  status ENUM(
-    'pending',
-    'started',
-    'complete_success',
-    'complete_failure',
-    'attempt_failed',
-    'dead') NOT NULL DEFAULT 'pending',
-
-  -- We can track this against a "max_attempt_count"
-  attempt_count INT(3) NOT NULL DEFAULT 0,
-
-  -- If there is a failure, tell the user why.
-  failure_reason VARCHAR(512) DEFAULT NULL,
-
-  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-
-  -- Failed jobs will set a next attempt time.
-  -- Subsequent tries can increase the timeout.
-  -- Failures because of permissions require human intervention => [retry_at=null].
-  -- Failures because of invalid files are dead => [status=dead].
-  retry_at TIMESTAMP NULL,
-
-  -- INDICES --
-  PRIMARY KEY (id),
-  UNIQUE KEY (token),
-  UNIQUE KEY (uuid_idempotency_token),
-  KEY fk_on_download_type (download_type),
-  KEY fk_creator_user_token (creator_user_token),
-  KEY index_status (status),
-  KEY index_creator_ip_address (creator_ip_address)
-
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_migrations/2022-08-05-202110_vocoder_models/up.sql

@@ -1,128 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-CREATE TABLE vocoder_models (
-  -- Not used for anything except replication.
-  id BIGINT(20) NOT NULL AUTO_INCREMENT,
-
-  -- Effective "primary key" (PUBLIC)
-  token VARCHAR(32) NOT NULL,
-
-  -- The architecture of the vocoder.
-  -- Currently supported values: 'hifigan'
-  vocoder_type VARCHAR(32) NOT NULL,
-
-  -- ========== MUTABLE METADATA ==========
-
-  -- The "name" of the vocoder, which might be complicated.
-  title VARCHAR(255) NOT NULL,
-
-  -- The description of the model in markdown.
-  description_markdown TEXT NOT NULL,
-
-  -- Generated HTML (not user-editable).
-  description_rendered_html TEXT NOT NULL,
-
-  -- ========== MUTABLE MOD-ONLY METADATA ==========
-
-  -- Whether the vocoder is recommended for use
-  is_staff_recommended BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- ========== IMMUTABLE PROVENANCE AND METADATA ==========
-
-  -- Where the file was originally downloaded (if it was downloaded)
-  original_download_url VARCHAR(512) DEFAULT NULL,
-
-  -- The filename that was used at upload time (if available)
-  original_filename VARCHAR(255) DEFAULT NULL,
-
-  -- File characteristics
-  file_size_bytes INT(10) NOT NULL DEFAULT 0,
-
-  -- ========== CREATOR DETAILS ==========
-
-  -- The person that created the template.
-  creator_user_token VARCHAR(32) NOT NULL,
-
-  -- For abuse tracking.
-  -- Wide enough for IPv4/6
-  creator_ip_address_creation VARCHAR(40) NOT NULL,
-  creator_ip_address_last_update VARCHAR(40) NOT NULL,
-
-  -- (THIS MIGHT NOT BE USED)
-  -- NB: DO NOT SORT!
-  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
-  creator_set_visibility ENUM(
-    'public',
-    'hidden',
-    'private'
-  ) NOT NULL DEFAULT 'public',
-
-  -- ========== BUCKET STORAGE ==========
-
-  -- The pytorch model
-  -- For now, this will be a hash of the file contents.
-  -- NB: NOT UNIQUE! We can allow duplicate uploads.
-  private_bucket_hash CHAR(64) NOT NULL,
-
-  -- The "full url" of the bucket path, not including the hostname
-  private_bucket_object_name VARCHAR(255) NOT NULL,
-
-  -- ========== MODERATION DETAILS ==========
-
-  -- Mods have locked the ability for non-authors (ie. people who didn't upload this)
-  -- to assign the vocoder for use with their TTS models.
-  -- TTS models that already incorporate the vocoder will not be affected.
-  -- The vocoder will remain visible.
-  -- This is independent of other visibility controls (is_mod_disabled_from_author_use,
-  -- deleted_at, and creator_set_visibility)
-  is_mod_disabled_from_public_use BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- Mods have locked the ability for the author (ie. the person that uploaded this)
-  -- to assign the vocoder for use with their TTS models.
-  -- TTS models that already incorporate the vocoder will not be affected.
-  -- The vocoder will remain visible.
-  -- This is independent of other visibility controls (is_mod_disabled_from_public_use,
-  -- deleted_at, and creator_set_visibility)
-  is_mod_disabled_from_author_use BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- Extremely popular models may be locked from deletion or modification by users.
-  -- If true, only moderators can edit or delete the vocoder.
-  is_mod_author_editing_locked BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- If a moderator has comments.
-  maybe_mod_comments VARCHAR(255) DEFAULT NULL,
-
-  -- The last moderator that made changes.
-  maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
-
-  -- ========== VECTOR CLOCK ==========
-
-  -- Incremented with every update.
-  version INT NOT NULL DEFAULT 0,
-
-  -- ========== TIMESTAMPS ==========
-
-  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-
-  -- If this is removed by a mod.
-  -- It completely disappears from the system.
-  user_deleted_at TIMESTAMP NULL,
-  mod_deleted_at TIMESTAMP NULL,
-
-  -- INDICES --
-  PRIMARY KEY (id),
-  UNIQUE KEY (token),
-  KEY fk_creator_user_token (creator_user_token),
-  KEY fk_maybe_mod_user_token (maybe_mod_user_token),
-  KEY index_creator_ip_address_creation (creator_ip_address_creation),
-  KEY index_creator_ip_address_last_update (creator_ip_address_last_update),
-  KEY index_creator_set_visibility (creator_set_visibility),
-  KEY index_private_bucket_hash (private_bucket_hash),
-  KEY index_is_mod_disabled_from_public_use (is_mod_disabled_from_public_use),
-  KEY index_is_mod_disabled_from_author_use (is_mod_disabled_from_author_use),
-  KEY index_is_mod_author_editing_locked (is_mod_author_editing_locked)
-
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_migrations_squashed/tts_tables/tts_inference_jobs.sql

@@ -1,124 +0,0 @@
--- NB: This is a manually squashed view of all the CREATE and ALTER statements,
--- with comments attached to the fields for centralized documentation.
-
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-CREATE TABLE tts_inference_jobs (
-  id BIGINT(20) NOT NULL AUTO_INCREMENT,
-
-  -- Effective "primary key" (PUBLIC)
-  -- This is so the in-progress results can be looked up by the UI.
-  token VARCHAR(32) NOT NULL,
-
-  -- Idempotency token from client
-  -- This is so the frontend client doesn't submit duplicate jobs.
-  uuid_idempotency_token VARCHAR(36) NOT NULL,
-
-  -- ========== SUCCESS CASE ==========
-
-  -- If the job completes successfully, this is the result token.
-  -- This is only populated on a successful result.
-  on_success_result_token VARCHAR(32) DEFAULT NULL,
-
-  -- ========== INFERENCE DETAILS ==========
-
-  -- The model to use.
-  -- This also determines which architecture we're using.
-  model_token VARCHAR(32) NOT NULL,
-
-  -- If set, use a different pretrained vocoder.
-  maybe_override_pretrained_vocoder VARCHAR(64) DEFAULT NULL,
-
-  -- The raw, unprocessed user input.
-  raw_inference_text TEXT NOT NULL,
-
-  -- ========== CREATOR DETAILS ==========
-
-  -- Foreign key to user
-  -- If no user is logged in, this is null.
-  maybe_creator_user_token VARCHAR(32) DEFAULT NULL,
-
-  -- Based on a cookie sent by the frontend.
-  -- We'll save this even if the user is logged in.
-  maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL,
-
-  -- For abuse tracking.
-  -- Wide enough for IPv4/6
-  creator_ip_address VARCHAR(40) NOT NULL,
-
-  -- (THIS MIGHT NOT BE USED)
-  -- NB: DO NOT SORT!
-  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
-  creator_set_visibility ENUM(
-    'public',
-    'hidden',
-    'private'
-  ) NOT NULL DEFAULT 'public',
-
-  -- ========== SOURCE METADATA ==========
-
-  is_from_api BOOLEAN NOT NULL DEFAULT FALSE,
-  is_for_twitch BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- If true, the request gets routed to a special "debug" worker.
-  is_debug_request BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- ========== JOB SYSTEM DETAILS ==========
-
-  -- Jobs begin as "pending", then transition to other states.
-  --
-  --  * Pending = job is ready to go
-  --  * Started = job is running
-  --  * Complete_Success = job is done (success)
-  --  * Complete_Failure = job is done (failure)
-  --  * Attempt_Failed = job failed but may retry.
-  --  * Dead = job failed permanently.
-  --
-  -- Pending -> Started -> Complete_Success
-  --                    |-> Complete_Failure
-  --                    \-> Attempt_Failed -> Started -> { Complete, Failed, Dead }
-  status ENUM(
-      'pending',
-      'started',
-      'complete_success',
-      'complete_failure',
-      'attempt_failed',
-      'dead') NOT NULL DEFAULT 'pending',
-
-  -- Priority *increases*, so a level of 2 will be higher than 1.
-  -- These are the level semantics currently:
-  --   - All jobs from anonymous FakeYou users have level 0.
-  --   - All jobs from logged in FakeYou users have level 1.
-  --   - All jobs from Twitch TTS (unpaid) have level 10 (ten).
-  --   - (There will be future levels for paid Twitch and social FakeYou rewards.)
-  priority_level TINYINT UNSIGNED NOT NULL DEFAULT 0,
-
-  -- We can track this against a "max_attempt_count"
-  attempt_count INT(3) NOT NULL DEFAULT 0,
-
-  -- If there is a failure, tell the user why.
-  failure_reason VARCHAR(512) DEFAULT NULL,
-
-  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-
-  -- Failed jobs will set a next attempt time.
-  -- Subsequent tries can increase the timeout.
-  -- Failures because of permissions require human intervention => [retry_at=null].
-  -- Failures because of invalid files are dead => [status=dead].
-  retry_at TIMESTAMP NULL,
-
-  -- INDICES --
-  PRIMARY KEY (id),
-  UNIQUE KEY (token),
-  UNIQUE KEY (uuid_idempotency_token),
-  KEY fk_on_success_result_token (on_success_result_token),
-  KEY fk_model_token (model_token),
-  KEY fk_maybe_creator_user_token (maybe_creator_user_token),
-  KEY index_status (status),
-  KEY index_creator_ip_address (creator_ip_address),
-  KEY index_is_debug_request (is_debug_request)
-
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_migrations_squashed/tts_tables/tts_models.sql

@@ -1,217 +0,0 @@
--- NB: This is a manually squashed view of all the CREATE and ALTER statements,
--- with comments attached to the fields for centralized documentation.
-
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-CREATE TABLE tts_models (
-  -- Not used for anything except replication.
-  id BIGINT(20) NOT NULL AUTO_INCREMENT,
-
-  -- Effective "primary key" (PUBLIC)
-  token VARCHAR(32) NOT NULL,
-
-  -- NB: DO NOT CHANGE ORDER; APPEND ONLY!
-  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
-  tts_model_type ENUM(
-      'not-set',
-      'tacotron2',
-      'glowtts',
-      'glowtts-vocodes',
-      'talknet'
-  ) NOT NULL DEFAULT 'not-set',
-
-  -- The name of the type of pipeline to use, eg "legacy_fakeyou", "legacy_vocodes", "english_v1", etc.
-  -- New pipelines may be added at any point in the future and the source of truth for them will mostly
-  -- live in the `storyteller-ml` repository.
-  -- Older models and brand new models will have this field set to null, and we'll infer which value to
-  -- lazily backfill based on the creation date and language setting.
-  text_pipeline_type VARCHAR(64) DEFAULT NULL,
-
-  -- For models distributed as archives/bundles (eg. TalkNet), we can pack
-  -- in a vocoder. Note that this vocoder does not have to be used.
-  has_self_contained_vocoder BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- For models distributed as archives/bundles (eg. TalkNet), we can pack
-  -- in a duration model.
-  has_self_contained_duration_model BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- For models distributed as archives/bundles (eg. TalkNet), we can pack
-  -- in a pitch model.
-  has_self_contained_pitch_model BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- NB(2022-07-05): THIS IS NOW MEANINGLESS AND SUPERSEDED BY `TEXT_PIPELINE_TYPE`.
-  -- NB: DO NOT CHANGE ORDER; APPEND ONLY!
-  -- How text should be handled.
-  text_preprocessing_algorithm ENUM(
-      'basic',
-      'arpabet'
-  ) NOT NULL DEFAULT 'basic',
-
-  -- If set, use a different pretrained vocoder.
-  -- If not set, use the website default. (Currently HifiGan-SS)
-  maybe_default_pretrained_vocoder VARCHAR(64) DEFAULT NULL,
-
-  -- If set, we'll use a custom vocoder (from table vocoder_models) instead of a default vocoder.
-  -- If not set, we'll use `maybe_default_pretrained_vocoder` (or the website default).
-  maybe_custom_vocoder_token VARCHAR(32) DEFAULT NULL,
-
-  -- Optional Pointer to a newer version of the voice
-  -- If there's a newer version, we can disable this one.
-  -- maybe_updated_model_token VARCHAR(32) DEFAULT NULL,
-
-  -- We can set as a combination of ['username' + 'voice-name']
-  -- There can be public aliases for voices, eg. a voice's default model.
-  -- A user can change this.
-  -- As such, these should not be foreign keys.
-  -- updatable_slug VARCHAR(64) NOT NULL,
-
-  -- Can be linked to a well-known voice
-  -- maybe_subject_token VARCHAR(32) DEFAULT NULL,
-  -- maybe_actor_subject_token VARCHAR(32) DEFAULT NULL,
-
-  -- ========== UNSTRUCTURED METADATA ==========
-
-  -- The full IETF BCP47 language tag (eg. en, en-US, es-419, ja-JP, pt, etc.)
-  ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en',
-
-  -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es")
-  ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en',
-
-  -- The "name" of the voice model, which might be complicated.
-  -- If maybe_subject_token (etc.) is set, then it's authoritative instead.
-  title VARCHAR(255) NOT NULL,
-
-  -- The description of the model in markdown.
-  description_markdown TEXT NOT NULL,
-
-  -- Generated HTML (not user-editable).
-  description_rendered_html TEXT NOT NULL,
-
-  -- eg. Bill Clinton, Batman, or Goku.
-  voice_name VARCHAR(255) DEFAULT NULL,
-
-  -- Optional; eg. Sean Schemmel
-  actor_name VARCHAR(255) DEFAULT NULL,
-
-  -- Optional; eg. if the voice is "happy" or a singer "a-capella", etc.
-  characteristic VARCHAR(255) DEFAULT NULL,
-
-  -- Optional; eg. if you train the model more than once.
-  version_string VARCHAR(255) DEFAULT NULL,
-
-  -- Whether the voice features on the front of FakeYou or Storyteller Twitch TTS
-  is_front_page_featured BOOLEAN NOT NULL DEFAULT FALSE,
-  is_twitch_featured BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- A suggested bot command prefix, eg. 'sonic' that could be interpreted as '/sonic'
-  -- by a TTS system. These are unique strings that can only be applied to one model at
-  -- a time at most. Only moderators can set this, and it's only for the best voices.
-  maybe_suggested_unique_bot_command VARCHAR(255) DEFAULT NULL,
-
-  -- ========== IMMUTABLE PROVENANCE AND METADATA ==========
-
-  -- Where the file was originally downloaded (if it was downloaded)
-  original_download_url VARCHAR(512) DEFAULT NULL,
-
-  -- The filename that was used at upload time (if available)
-  original_filename VARCHAR(255) DEFAULT NULL,
-
-  -- File characteristics
-  file_size_bytes INT(10) NOT NULL DEFAULT 0,
-
-  -- ========== CREATOR DETAILS ==========
-
-  -- The person that created the template.
-  creator_user_token VARCHAR(32) NOT NULL,
-
-  -- For abuse tracking.
-  -- Wide enough for IPv4/6
-  creator_ip_address_creation VARCHAR(40) NOT NULL,
-  creator_ip_address_last_update VARCHAR(40) NOT NULL,
-
-  -- (THIS MIGHT NOT BE USED)
-  -- NB: DO NOT SORT!
-  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
-  creator_set_visibility ENUM(
-    'public',
-    'hidden',
-    'private'
-   ) NOT NULL DEFAULT 'public',
-
-  -- ========== BUCKET STORAGE ==========
-
-  -- The pytorch model
-  -- For now, this will be a hash of the file contents.
-  -- NB: NOT UNIQUE! We can allow duplicate uploads.
-  private_bucket_hash CHAR(64) NOT NULL,
-  -- The "full url" version of the path
-  private_bucket_object_name VARCHAR(255) NOT NULL,
-
-  -- Whether the file is a zip archive (rather than a model file).
-  private_bucket_object_is_archive BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- ========== RATING AND RANKING ==========
-
-  -- Calculated average, on a scale of 0-100
-  -- Null with zero ratings.
-  -- calculated_average_score INT(3) DEFAULT NULL,
-
-  -- Other metrics
-  -- calculated_total_ratings_submitted_count INT(10) NOT NULL DEFAULT 0,
-  -- calculated_total_uses_count BIGINT(10) NOT NULL DEFAULT 0,
-
-  -- ========== MODERATION DETAILS ==========
-
-  -- Mods have to approve of tts models for them to show up in a public index.
-  -- Models can be used by the author (or other parties that know the URL) before
-  -- they're approved, but unapproved models won't show up in public indices.
-  -- This is independent of other visibility controls (is_mod_disabled, deleted_at,
-  -- and creator_set_visibility)
-  is_public_listing_approved BOOLEAN DEFAULT NULL,
-
-  -- Extremely popular models may be locked from deletion or modification by users.
-  is_locked_from_user_modification BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- In this case, a moderator disables it from being used for inference.
-  -- This also disables it for the creator.
-  -- Unlike deletion, it remains "visible" to those that have access.
-  is_locked_from_use BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- If a moderator has comments.
-  maybe_mod_comments VARCHAR(255) DEFAULT NULL,
-
-  -- The last moderator that made changes.
-  maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
-
-  -- ========== VECTOR CLOCK ==========
-
-  -- Incremented with every update.
-  version INT NOT NULL DEFAULT 0,
-
-  -- ========== TIMESTAMPS ==========
-
-  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-
-  -- If this is removed by a mod.
-  -- It completely disappears from the system.
-  user_deleted_at TIMESTAMP NULL,
-  mod_deleted_at TIMESTAMP NULL,
-
-  -- INDICES --
-  PRIMARY KEY (id),
-  UNIQUE KEY (token),
-  KEY fk_creator_user_token (creator_user_token),
-  KEY fk_maybe_mod_user_token (maybe_mod_user_token),
-  KEY fk_maybe_custom_vocoder_token (maybe_custom_vocoder_token),
-  KEY index_creator_ip_address_creation (creator_ip_address_creation),
-  KEY index_creator_ip_address_last_update (creator_ip_address_last_update),
-  KEY index_creator_set_visibility (creator_set_visibility),
-  KEY index_private_bucket_hash (private_bucket_hash),
-  KEY index_is_public_listing_approved (is_public_listing_approved),
-  KEY index_is_locked_from_user_modification (is_locked_from_user_modification),
-  KEY index_is_locked_from_use (is_locked_from_use)
-
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_sql/actions/generic_download_jobs.sql

@@ -0,0 +1,48 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- 1) Find uploads that failed (recently)
+
+select
+    id,
+    token,
+    download_type,
+    title,
+    download_url,
+    status,
+    attempt_count
+from generic_download_jobs
+where status IN ('started', 'attempt_failed', 'complete_failure', 'dead')
+and download_type NOT IN ('vits', 'hifigan')
+and created_at > NOW() - INTERVAL 15 DAY;
+
+
+-- 2) Then restart them
+-- NB: Double inner query is to get away from query optimizer:
+--  "You can't specify target table 'generic_download_jobs' for update in FROM clause"
+
+update generic_download_jobs
+set
+    status='pending',
+    attempt_count=0
+where token IN (
+    select token from (
+        select token
+        from generic_download_jobs
+        where status IN ('started', 'attempt_failed', 'complete_failure', 'dead')
+        and download_type NOT IN ('vits', 'hifigan')
+        and created_at > NOW() - INTERVAL 15 DAY
+    ) as x
+);
+
+
+-- Kill bad pending downloads
+-- eg. awful voices, like Hitler
+
+update generic_download_jobs
+set
+    status='dead',
+    download_url = CONCAT('NOPE', download_url)
+where token = 'jdown_ssape40dxbfgvhtsmedt34418x'
+limit 1;

_sql/actions/generic_inference_jobs.sql

@@ -0,0 +1,13 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Kill routed jobs
+update generic_inference_jobs set status = 'dead' where maybe_routing_tag IS NOT NULL limit 100;
+
+-- Get pending jobs
+select count(*) from generic_inference_jobs where status = 'pending';
+
+update generic_inference_jobs
+set status = 'dead'
+where status IN ('pending', 'started', 'attempt_failed');

_sql/actions/tts_inference_jobs.sql

@@ -0,0 +1,24 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Kill pending jobs
+update tts_inference_jobs set status = 'dead' where status = 'pending';
+
+-- Kill all low priority pending, waiting, and in-progress jobs
+update tts_inference_jobs
+set status = 'dead'
+where status IN ('pending', 'started', 'attempt_failed')
+  and priority_level IN (0, 1);
+
+-- Kill all pending jobs
+update tts_inference_jobs
+set status = 'dead'
+where status = 'pending';
+
+-- Kill all waiting and in-progress jobs
+update tts_inference_jobs
+set status = 'dead'
+where status IN ('pending', 'started', 'attempt_failed');
+
+

_sql/actions/tts_results.sql

@@ -0,0 +1,11 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Delete TTS results for a single model
+-- Do this in short batches so a lock isn't held for prohibitively long.
+update tts_results
+set mod_deleted_at = NOW()
+where model_token = 'TM:ztt5s1be5tq6'
+  and mod_deleted_at IS NULL
+    limit 5000;

_sql/actions/users.sql

@@ -0,0 +1,35 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Upgrade top TTS model uploaders to a loyalty plan
+update users
+set maybe_loyalty_program_key = 'fakeyou_contributor'
+where token IN (
+    select creator_user_token from
+        (
+            select count(*) as upload_count,
+                   creator_user_token
+            from tts_models
+            group by creator_user_token
+            having upload_count >= 10
+            order by upload_count desc
+        ) as t
+);
+
+-- Upgrade top VC model uploaders to a loyalty plan
+update users
+set maybe_loyalty_program_key = 'fakeyou_contributor'
+where token IN (
+    select creator_user_token from
+        (
+            select count(*) as upload_count,
+                   creator_user_token
+            from voice_conversion_models
+            group by creator_user_token
+            having upload_count >= 10
+            order by upload_count desc
+        ) as t
+);
+
+

_sql/actions/voice_conversion_models.sql

@@ -0,0 +1,312 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Remove temporary test models
+update voice_conversion_models
+set
+    mod_deleted_at = NOW(),
+    maybe_mod_comments = 'temporary test model'
+where token IN (
+    'vcm_jt9mdz9xjjcb',
+    'vcm_4xzkzdhs4kk5',
+    'vcm_502wkt2pca3y'
+);
+
+-- Remove broken models
+update voice_conversion_models
+set
+    mod_deleted_at = NOW(),
+    maybe_mod_comments = 'broken model'
+where token IN (
+    'vcm_982y9g6v3jrg',
+    'vcm_16rpyehf7f0a',
+    'vcm_8vjtz3dbnw17',
+    'vcm_tc9mwr60cjth',
+    'vcm_bshsyp01w9vg',
+    'vcm_31whj8apmqj7',
+    'vcm_84qa0an44p8p',
+    'vcm_9xx4v5qz5fbg',
+    'vcm_7zp2b2hs2at2',
+    'vcm_zhvk0mnnb7zp',
+    'vcm_1vc4scq9qp6m',
+    'vcm_sr1b54mwy0ym',
+    'vcm_qx96y8aasydk'
+);
+
+-- Remove duplicate models
+update voice_conversion_models
+set
+    mod_deleted_at = NOW(),
+    maybe_mod_comments = 'duplicate model'
+where token IN (
+    'vcm_6w21em1bs7q3',
+    'vcm_sbhvyj7926w4',
+    'vcm_anv469g89adz',
+    'vcm_xsx5yx585c7x',
+    'vcm_dpahzjjpkf18',
+    'vcm_y09dqe6zgwyk',
+    'vcm_xrp96dwgppv2',
+    'vcm_84awjbjxts6n',
+    'vcm_b2x2sn8kf28j',
+    'vcm_41nw6xgr5x3v',
+    'vcm_a6x83tk0ttf2',
+    'vcm_dgjgqxq5w97j',
+    'vcm_59k5j452x7xy',
+    'vcm_09p6vqnk7n05',
+    'vcm_attanwh383hg',
+    'vcm_t8d4f6w9pa8t',
+    'vcm_s8s4q4c65xsy',
+    'vcm_6e2av8pmkyz5',
+    'vcm_82x5f330z4wf',
+    'vcm_z6bax4kvjdx6',
+    'vcm_v8hf35cyq3ft',
+    'vcm_1h9b4v6nxmf5',
+    'vcm_781bzn81qf6k',
+    'vcm_w0xjjsexktby',
+    'vcm_awsq3q4c711w',
+    'vcm_kttxt7wgb6s6',
+    'vcm_ets26z5fpt8d',
+    'vcm_wjrw59wmwcrr',
+    'vcm_bebgka17n0sr',
+    'vcm_6mj0x3qd7yfd',
+    'vcm_b71ke4fm61nx',
+    'vcm_srz3gas5ra4d',
+    'vcm_k64jj2t2j05b',
+    'vcm_140rd035642m',
+    'vcm_5y93wgr90s91',
+    'vcm_jwp6p08byjn9',
+    'vcm_ets26z5fpt8d',
+    'vcm_7yfwzsyvrpq4',
+    'vcm_f1vq0hp0tes2',
+    'vcm_9f05kg36q4fg',
+    'vcm_1xs9c2sgykgy',
+    'vcm_hnd9z2fxq661',
+    'vcm_jxd74xhcszry',
+    'vcm_tvk0g6dwg3yv',
+    'vcm_vrsdv4ahterx',
+    'vcm_e2debszbm8vt',
+    'vcm_rh98fw9a9an1',
+    'vcm_2bvd1gmysaxf',
+    'vcm_2cqq1j33y2hr',
+    'vcm_ysj7kwb96kcp',
+    'vcm_ef0eaneyr72p',
+    'vcm_t59d7vdxtqs4',
+    'vcm_90hrepevshtg',
+    'vcm_4st9ex13vnbh',
+    'vcm_30fne697qhab',
+    'vcm_qy7evy03p6tg',
+    'vcm_775eze19tv0n',
+    'vcm_4bzkfvaexbv3',
+    'vcm_08v2tdga8v93',
+    'vcm_whxeyxzw12q6',
+    'vcm_0rj5b6npxfkh',
+    'vcm_hv22emyfkex4',
+    'vcm_a1fmtgthezs0',
+    'vcm_z9fpz1p5bpr2',
+    'vcm_2y8ck2dbppkq',
+    'vcm_0c9h3mzc4kre',
+    'vcm_m0z3tt4gytm9'
+);
+
+-- Remove duplicate models (2)
+update voice_conversion_models
+set
+    mod_deleted_at = NOW(),
+    maybe_mod_comments = 'duplicate model'
+where token IN (
+    'vcm_v790w43dv3ce',
+    'vcm_xmzm7eextc3n',
+    'vcm_r3b91dr0eta5',
+    'vcm_p1rpmfmckd5c',
+    'vcm_cpajdgenjfw7',
+    'vcm_qchjs52fsr9y',
+    'vcm_pkkjy8c6cpt3',
+    'vcm_f8ra6j8d6wgv',
+    'vcm_mhsz9bjn19tw',
+    'vcm_btd5zab2jdqv',
+    'vcm_q5evptaaqne3',
+    'vcm_wqtqs4gj2zmj',
+    'vcm_14rv35fct2ay',
+    'vcm_418j0h41qy8a',
+    'vcm_3m5m6bpm0rt2',
+    'vcm_gdw4krmaxpza',
+    'vcm_b3xhdb8zymvf',
+    'vcm_aak9m6nvj3am',
+    'vcm_mcw5kn21bbjh',
+    'vcm_fydr6ts21mtf',
+    'vcm_zpw560jzyqpw',
+    'vcm_w24wcctnxbxf',
+    'vcm_ya5tpzdcvvcz',
+    'vcm_1ctmgrjtn1j7',
+    'vcm_rk8fycc0reb2',
+    'vcm_e2c8942vky34',
+    'vcm_wevg8cf3hcw8',
+    'vcm_6hzbyg453xb9',
+    'vcm_xqfffshe2g0q',
+    'vcm_32j1zmcvn61n'
+);
+
+-- Remove duplicate models (3)
+update voice_conversion_models
+set
+    mod_deleted_at = NOW(),
+    maybe_mod_comments = 'duplicate model'
+where token IN (
+    'vcm_46sbq0r9hfjk',
+    'vcm_k8nseqy8jyte',
+    'vcm_gs8qam6npe46',
+    'vcm_djh15g49cxm8',
+    'vcm_d7zygjnp35my',
+    'vcm_334yqeq11rpm',
+    'vcm_6m3b6121318k',
+    'vcm_a419w7gw6jrt',
+    'vcm_wtyszwjwq0ts',
+    'vcm_0q9t2azhqvf1',
+    'vcm_fx2k40ffzjqg',
+    'vcm_wvjqmzd8y4mt',
+    'vcm_b2635v3p50je',
+    'vcm_zfyr07z2d0rb',
+    'vcm_yn2sn5j2jwed',
+    'vcm_z596ja3yvyy9',
+    'vcm_a1p854bn1sjs',
+    'vcm_fv24vmr93805',
+    'vcm_0y7vsc2e9s3r',
+    'vcm_5aqzq0d246rm',
+    'vcm_trvge62x7je9',
+    'vcm_sghga68kwggk',
+    'vcm_raxcc33pj608',
+    'vcm_a66b3va0sj64',
+    'vcm_bd1rjawqypw6',
+    'vcm_evzff14n721p',
+    'vcm_jnrr4ht9f42f'
+);
+
+-- Remove duplicate models (4)
+update voice_conversion_models
+set
+    mod_deleted_at = NOW(),
+    maybe_mod_comments = 'duplicate model'
+where token IN (
+    'vcm_ds2pz63sagan',
+    'vcm_wkczgvq2fjqq',
+    'vcm_z8137w1rddse',
+    'vcm_gknzxybf4erw',
+    'vcm_z0m1qtmawccn',
+    'vcm_yek6pmgvbb1s',
+    'vcm_qqhnjwsjrvnx',
+    'vcm_8fetm7qz94cp',
+    'vcm_kz1zm1d269j0',
+    'vcm_rfnkc8g5gbbv',
+    'vcm_4ncqxetfe7dm',
+    'vcm_qkpfem2a153n',
+    'vcm_z5jkd9r4cawf',
+    'vcm_vgh6ja6zzj0p',
+    'vcm_bd473tdrrzhk',
+    'vcm_2k7tmx3ndd3y',
+    'vcm_weyxz516dd2m',
+    'vcm_1j4kpjvgsj6k',
+    'vcm_211g17ea44fw',
+    'vcm_7zkbj6be0yr5',
+    'vcm_nnaaxeqy4xj6',
+    'vcm_wp7sgtz6mhc4',
+    'vcm_h5w7fzhxcpxm',
+    'vcm_7d5rxpqkjy07',
+    'vcm_9vhjvrptygfz',
+    'vcm_4qbbawahnkvb',
+    'vcm_am870m8162at',
+    'vcm_cftdb5227qt5',
+    'vcm_m7vv1mcxfxvm',
+    'vcm_qn8vsvvh1rqc',
+    'vcm_erhvqjvk1dsk',
+    'vcm_4zs8tdrw971e',
+    'vcm_4zs8tdrw971e',
+    'vcm_z8htjaejr1wx',
+    'vcm_99wdsazefjwr',
+    'vcm_k1at1sbepvqv',
+    'vcm_xcp34f5d7ygs',
+    'vcm_8v8srq9sx7cq',
+    'vcm_f2yf7bxmk8h5',
+    'vcm_gvtrjte4as1s',
+    'vcm_2s0y50avy43b',
+    'vcm_r638ygfxkpmq',
+    'vcm_1ttm6pxcpqna',
+    'vcm_0ytfkxh6pm4f',
+    'vcm_zderdrfmgq6c',
+    'vcm_jvdpr6mtddb8',
+    'vcm_9tvcv961g09x',
+    'vcm_aynpmxbhpy7n',
+    'vcm_2gzsjb5bdsdq',
+    'vcm_a0vep64tz56n',
+    'vcm_zt4hgc9pyv11',
+    'vcm_szfxm0crd8y8',
+    'vcm_3q89kba3cy5z',
+    'vcm_ywpwnhk4pv4x',
+    'vcm_4he8jjmengb3',
+    'vcm_jpdfswn0empy',
+    'vcm_2nztkt3rgm1q',
+    'vcm_q5gyxf5a11zf',
+    'vcm_sa7k9ycne147',
+    'vcm_p6gxtj9qefsg',
+    'vcm_j2fvzdyachtw',
+    'vcm_ecbv0rhc6br3',
+    'vcm_jrgwspz2wt7p',
+    'vcm_f5b4j3rt180n',
+    'vcm_pzajxdfcjfpa',
+    'vcm_rq4f2cbbm6j1',
+    'vcm_rdmfpbnab47w',
+    'vcm_03dtk4xtggm6',
+    'vcm_zx7mdkh9s9qb',
+    'vcm_e0megdtf71x6'
+);
+
+-- Remove older models replaced by new versions
+update voice_conversion_models
+set
+    mod_deleted_at = NOW(),
+    maybe_mod_comments = 'replaced with new version'
+where token IN (
+    'vcm_gdg3najz6j3g'
+);
+
+-- Remove takedown request models
+update voice_conversion_models
+set
+    mod_deleted_at = NOW(),
+    maybe_mod_comments = 'takedown request'
+where token IN (
+    'vcm_2af74np3jh6q',
+    'vcm_8kk7355dc184',
+    'vcm_nsh89pcx4569',
+    'vcm_q7ewn3yym4gy',
+    'vcm_cwngb21xsf66',
+    'vcm_jmzjrky179cm',
+    'vcm_hj6a1nt1yvh9',
+    'vcm_6wwx1e3cd48s',
+    'vcm_pc76gc1xh7jg',
+    'vcm_gb0p1wmtk9wp',
+    'vcm_pbs955dvaeq6',
+    'vcm_x2wyzr6h6539',
+    'vcm_py7qwz310qff',
+    'vcm_k01bqvh0dseq',
+    'vcm_5m4dsn2fwa0q',
+    'vcm_hn9m98g1am3z',
+    'vcm_f9168tyyn6ex',
+    'vcm_03y9w4h99r1h',
+    'vcm_yr9f1s0xc7pc',
+    'vcm_45767r26a9bb',
+    'vcm_rq3cvp3x3hnh',
+    'vcm_ng1nz8d67z4v',
+    'vcm_f5arabzrk4en',
+    'vcm_d379600cr8yx',
+    'vcm_p7m9jw1krrkr',
+    'vcm_x7v3m9mcvten',
+    'vcm_6r057m7594bv',
+    'vcm_wkm9e42d3spm'
+);
+
+
+
+
+
+

_sql/actions/w2l_inference_jobs.sql

@@ -0,0 +1,25 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Kill pending jobs
+update w2l_inference_jobs set status='dead' where status IN ('pending', 'started', 'attempt_failed');
+
+-- Unstick old jobs that might be stuck as "started"
+update w2l_template_upload_jobs
+set status='dead'
+where status='started'
+  and created_at < ( CURDATE() - INTERVAL 1 DAY );
+
+-- Kill pending jobs with tiktok URLs:
+-- These need to be blocked in the server and job directly.
+update w2l_template_upload_jobs
+set status='dead'
+where status='pending'
+  and download_url LIKE '%tiktok%';
+
+
+-- Kill all pending jobs
+update w2l_template_upload_jobs
+set status='dead'
+where status IN ('pending', 'started', 'attempt_failed');

_sql/analytics/analytics_stripe_sigma/basic_user_queries.sql

@@ -0,0 +1,36 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- NB: These are for use in Stripe Sigma, not MySQL
+-- NB: You must use single quotes instead of double quotes when comparing string values.
+
+-- Histogram of customers by country
+select address_country, count(*)
+from customers
+group by address_country
+
+select customer_id, subscriptions.status
+from subscriptions
+
+
+select distinct customer_id
+from subscriptions
+where subscriptions.status = 'active'
+or subscriptions.status = 'past_due'
+
+select address_country, count(*) as country_count
+from customers
+where id IN (
+    select distinct customer_id
+    from subscriptions
+    where subscriptions.status = 'active'
+    or subscriptions.status = 'past_due'
+)
+group by address_country
+
+
+-- Histogram
+select address_country, count(*)
+from customers
+group by address_country

_sql/analytics/analytics_stripe_sigma/subscriptions.sql

@@ -0,0 +1,90 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- NB: These are for use in Stripe Sigma, not MySQL
+-- NB: You must use single quotes instead of double quotes when comparing string values.
+
+-- Status values: past_due, incomplete, active, canceled, incomplete_expired (perhaps not exhaustive)
+select distinct status
+from subscriptions
+
+select count(*)
+from subscriptions
+where status IN ('active');
+
+-- Metadata is in the 'subscriptions_metadata' table, and we store 'username', 'email', and 'user_token'
+select
+    m.value,
+    m.key,
+    s.id,
+    s.created,
+    s.cancel_at,
+    s.canceled_at
+from subscriptions as s
+join subscriptions_metadata as m
+on m.subscription_id = s.id
+where m.key = 'username'
+
+
+-- Full mapping of Stripe subscribers to FakeYou users
+select
+    s.id as subscription_id,
+    user_tokens.user_token,
+    usernames.username,
+    emails.email,
+    s.status as subscription_status,
+    s.created,
+    s.cancel_at,
+    s.canceled_at
+from subscriptions as s
+join (
+    select
+        m.subscription_id,
+        m.value as username
+    from subscriptions_metadata as m
+    where m.key = 'username'
+) as usernames
+on s.id = usernames.subscription_id
+join (
+    select m.subscription_id,
+           m.value as email
+    from subscriptions_metadata as m
+    where m.key = 'email'
+) as emails
+on s.id = emails.subscription_id
+join (
+    select m.subscription_id,
+           m.value as user_token
+    from subscriptions_metadata as m
+    where m.key = 'user_token'
+) as user_tokens
+on s.id = user_tokens.subscription_id
+order by s.id asc
+
+-- Without emails
+select
+    s.id as subscription_id,
+    user_tokens.user_token,
+    usernames.username,
+    s.status as subscription_status,
+    s.created,
+    s.cancel_at,
+    s.canceled_at
+from subscriptions as s
+join (
+    select
+        m.subscription_id,
+        m.value as username
+    from subscriptions_metadata as m
+    where m.key = 'username'
+) as usernames
+on s.id = usernames.subscription_id
+join (
+    select m.subscription_id,
+           m.value as user_token
+    from subscriptions_metadata as m
+    where m.key = 'user_token'
+) as user_tokens
+on s.id = user_tokens.subscription_id
+order by s.id asc

_sql/analytics/analytics_stripe_sigma/user_cohorts.sql

@@ -0,0 +1,8 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- NB: These are for use in Stripe Sigma, not MySQL
+-- NB: You must use single quotes instead of double quotes when comparing string values.
+
+

_sql/analytics/generic_inference/generic_inference_jobs.sql

@@ -0,0 +1,196 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- See how long recent jobs are taking
+select
+    id,
+    maybe_input_source_token,
+    assigned_worker,
+    assigned_cluster,
+    success_execution_millis / 1000 / 60 as minutes
+from generic_inference_jobs
+where status != 'pending'
+and success_execution_millis IS NOT NULL
+order by id desc
+limit 50;
+
+
+-- Detailed report on most recent jobs, ordered by worst performing.
+-- TODO: Determine if the problem is in downloading models. Make sure the job timer doesn't include
+--   sections where models / files get downloaded.
+select *
+from (
+         select
+             jobs.created_at,
+             jobs.first_started_at,
+             jobs.id,
+             jobs.status,
+             jobs.attempt_count as attempts,
+             trim(replace(replace(jobs.internal_debugging_failure_reason, "\n", ""), "\r", ""))
+                 as i_failure_reason,
+             -- u.username,
+             jobs.assigned_worker,
+             jobs.assigned_cluster,
+             jobs.maybe_input_source_token,
+             jobs.maybe_model_type,
+             jobs.maybe_model_token,
+             m.media_source,
+             m.maybe_original_mime_type as mime_type,
+             TRUNCATE(jobs.success_execution_millis / 1000 / 60, 2) as execution_mins,
+             TRUNCATE(jobs.success_inference_execution_millis / 1000 / 60, 2) as inference_mins,
+             TRUNCATE((jobs.success_execution_millis - jobs.success_inference_execution_millis) / 1000 / 60, 2) as extra_mins,
+             TRUNCATE(m.original_duration_millis / 1000 / 60, 2) as input_mins,
+             jobs.success_execution_millis / m.original_duration_millis as ratio
+         from generic_inference_jobs AS jobs
+                  left join users AS u on
+                 u.token = jobs.maybe_creator_user_token
+                  left join media_uploads AS m on
+                 m.token = jobs.maybe_input_source_token
+         where
+             jobs.status != 'pending'
+        AND jobs.created_at > NOW() - INTERVAL 20 MINUTE
+         order by id desc
+             limit 200
+     ) as t
+order by execution_mins desc;
+
+order by assigned_worker desc, first_started_at asc;
+
+order by first_started_at asc;
+
+order by assigned_worker desc, execution_mins desc;
+
+
+-- Get pending so-vits-svc jobs
+select count(*)
+from generic_inference_jobs
+where maybe_model_type = 'so_vits_svc'
+  and status='pending';
+
+
+-- Get jobs that have routing tags
+select
+    id,
+    maybe_input_source_token,
+    maybe_creator_user_token,
+    maybe_routing_tag,
+    status
+from generic_inference_jobs
+where maybe_routing_tag IS NOT NULL
+    limit 10;
+
+
+
+-- Debug that we're setting the correct metadata on jobs
+-- For some reasons storyteller-web is enqueuing the wrong type!
+select
+    jobs.id,
+    jobs.token,
+    jobs.maybe_model_type as jobs_model_type,
+    jobs.maybe_model_token as jobs_model_token,
+    models.token as model_token,
+    models.model_type as model_type
+from generic_inference_jobs as jobs
+left outer join voice_conversion_models as models
+on jobs.maybe_model_token = models.token
+order by jobs.id
+desc
+limit 100;
+
+-- Generic inference jobs + legacy TTS (this is a production endpoint)
+-- UPDATED QUERY: Queues as named rows
+        SELECT
+            maybe_model_type as queue_type,
+            count(*) as pending_job_count,
+            NOW() as present_time
+        FROM (
+            SELECT
+                token,
+                maybe_model_type
+            FROM generic_inference_jobs
+            WHERE status IN ("pending", "attempt_failed")
+            UNION
+            SELECT
+                token,
+                maybe_model_type
+            FROM generic_inference_jobs
+            WHERE status IN ("started")
+            AND created_at > (CURDATE() - INTERVAL 15 MINUTE)
+        ) as generic_inner
+        GROUP BY queue_type
+UNION
+        SELECT
+            "legacy_tts" as queue_type,
+            count(*) as pending_job_count,
+            NOW() as present_time
+        FROM
+        (
+             SELECT token
+             FROM tts_inference_jobs
+             WHERE status IN ("pending", "attempt_failed")
+             UNION
+             SELECT token
+             FROM tts_inference_jobs
+             WHERE status IN ("started")
+             AND created_at > (CURDATE() - INTERVAL 15 MINUTE)
+        ) as legacy_inner
+        GROUP BY queue_type;
+
+-- Generic inference jobs + legacy TTS (this was a production endpoint)
+-- OLD QUERY: Queues as columns
+SELECT
+(
+    SELECT
+        COUNT(distinct token) as generic_job_count
+    FROM
+        (
+            SELECT token
+            FROM generic_inference_jobs
+            WHERE status = "started"
+              AND created_at > (CURDATE() - INTERVAL 5 MINUTE)
+            UNION
+            SELECT token
+            FROM generic_inference_jobs
+            WHERE status IN ("pending", "attempt_failed")
+        ) as g
+) as generic_job_count,
+(
+    SELECT
+        COUNT(distinct token) as legacy_tts_job_count
+    FROM
+        (
+            SELECT token
+            FROM tts_inference_jobs
+            WHERE status = "started"
+              AND created_at > (CURDATE() - INTERVAL 5 MINUTE)
+            UNION
+            SELECT token
+            FROM tts_inference_jobs
+            WHERE status IN ("pending", "attempt_failed")
+        ) as t
+) as legacy_tts_job_count,
+NOW() as present_time;
+
+-- Histogram of inference
+select
+    date(created_at) as created_date,
+    count(*) as job_count
+from generic_inference_jobs
+group by created_date
+
+-- Histogram of inference (within range)
+select
+    date(created_at) as created_date,
+    count(*) as job_count
+from generic_inference_jobs
+where created_at > (CURDATE() - INTERVAL 5 DAY)
+group by created_date
+
+-- Histogram of inference (within range)
+select
+    date(created_at) as created_date,
+    count(*) as job_count
+from generic_inference_jobs
+where date(created_at) >= "2023-01-01"
+group by created_date

_sql/analytics/generic_inference/media_uploads.sql

@@ -0,0 +1,50 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+select
+    mu.id,
+    mu.token,
+    u.username,
+    mu.media_source,
+    maybe_original_filename as filename,
+    original_file_size_bytes as size_bytes,
+    original_duration_millis as duration_millis,
+    maybe_original_mime_type as mime_type
+from media_uploads as mu
+    left join users AS u
+        on u.token = mu.maybe_creator_user_token
+order by id desc limit 10;
+
+-- Analyze all media
+select
+    mu.id,
+    mu.token,
+    u.username,
+    mu.media_source,
+    maybe_original_filename as filename,
+    original_file_size_bytes as size_bytes,
+    original_duration_millis as duration_millis,
+    maybe_original_mime_type as mime_type
+from media_uploads as mu
+         left join users AS u
+                   on u.token = mu.maybe_creator_user_token
+order by id desc
+    limit 40;
+
+-- Analyze broken media (no duration)
+select
+    mu.id,
+    mu.token,
+    u.username,
+    mu.media_source,
+    maybe_original_filename as filename,
+    original_file_size_bytes as size_bytes,
+    original_duration_millis as duration_millis,
+    maybe_original_mime_type as mime_type
+from media_uploads as mu
+    left join users AS u
+        on u.token = mu.maybe_creator_user_token
+where mu.original_duration_millis = 0
+order by id desc
+limit 40;

_sql/analytics/tts/tts_category_assignments.sql

@@ -0,0 +1,43 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Select TTS models within a category
+-- (NOTE CATEGORIES ARE *SOFT DELETED* AND THIS MAY MIS-ATTRIBUTE)
+select
+    tts.token,
+    tts.ietf_language_tag,
+    tts.title
+from tts_category_assignments as cat
+join tts_models as tts
+on tts.token = cat.model_token
+where
+    cat.category_token = 'CAT:n06k9gfgq20';
+
+-- Select TTS models within a category
+select
+    tts.token,
+    tts.ietf_language_tag,
+    tts.title
+from tts_category_assignments as cat
+         join tts_models as tts
+              on tts.token = cat.model_token
+where
+        cat.category_token = 'CAT:jtdwpy6ptrn'
+AND cat.deleted_at IS NULL;
+
+
+-- Select TTS models within a category
+-- Exclude given languages.
+select
+    tts.token,
+    tts.ietf_language_tag,
+    tts.title
+from tts_category_assignments as cat
+         join tts_models as tts
+              on tts.token = cat.model_token
+where
+        cat.category_token = 'CAT:jtdwpy6ptrn'
+  AND tts.ietf_language_tag NOT IN ('es', 'es-419', 'es-ES', 'es-MX')
+  AND cat.deleted_at IS NULL;
+

_sql/analytics/tts/tts_inference_jobs.sql

@@ -0,0 +1,85 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Total pending jobs
+select count(*)
+from tts_inference_jobs
+where status = 'pending';
+
+-- Total pending API jobs
+select count(*)
+from tts_inference_jobs
+where status = 'pending'
+and is_from_api = true;
+
+-- Top IP addresses making requests
+select distinct creator_ip_address, count(*) as attempts
+from tts_inference_jobs
+where status = 'pending'
+group by creator_ip_address
+order by attempts desc
+limit 50;
+
+-- Top (IP addresses, usernames) making requests
+select distinct j.creator_ip_address, u.username, count(*) as attempts
+from tts_inference_jobs AS j
+left outer join users AS u
+on u.token = j.maybe_creator_user_token
+where j.status = 'pending'
+group by j.creator_ip_address, u.username
+order by attempts desc
+    limit 50;
+
+-- Find a job by an IP
+select *
+from tts_inference_jobs
+where status = 'pending'
+and creator_ip_address = '186.188.28.73'
+limit 1;
+
+-- Top voices in requests
+select distinct model_token, count(*) as attempts
+from tts_inference_jobs
+where status = 'pending'
+group by model_token
+order by attempts desc
+    limit 50;
+
+-- Sample the pending inference text
+select creator_ip_address,
+       model_token,
+       maybe_creator_user_token,
+       raw_inference_text
+from tts_inference_jobs
+where status = 'pending';
+
+-- Sample the pending inference text (truncated)
+select creator_ip_address,
+       model_token,
+       maybe_creator_user_token,
+       TRIM(REPLACE(SUBSTRING(raw_inference_text, 1, 50), '\n', ''))
+from tts_inference_jobs
+where status = 'pending'
+limit 100;
+
+-- Sample long pending texts
+select creator_ip_address,
+       model_token,
+       maybe_creator_user_token,
+       raw_inference_text
+from tts_inference_jobs
+where status = 'pending'
+and length(raw_inference_text) > 100
+limit 100;
+
+-- Delete old TTS inference jobs (1)
+DELETE FROM tts_inference_jobs
+WHERE status IN ('dead', 'complete_success', 'complete_failure')
+LIMIT 100000;
+
+-- Delete old TTS inference jobs (2)
+-- Roughly 20 seconds to delete 100k, 4 minutes for 1M.
+DELETE FROM tts_inference_jobs
+WHERE created_at < ( CURDATE() - INTERVAL 1 DAY )
+LIMIT 100000;

_sql/analytics/tts/tts_models.sql

@@ -0,0 +1,380 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+--
+-- Top 100 models all time by use count
+--
+select
+    m.token,
+    m.text_pipeline_type,
+    m.ietf_language_tag,
+    m.title,
+    u.username,
+    r.use_count,
+    m.created_at,
+    m.user_deleted_at,
+    m.mod_deleted_at
+from (
+         select model_token, count(*) as use_count
+         from tts_results
+         group by model_token
+     ) as r
+         join tts_models as m
+              on m.token = r.model_token
+         join users as u
+              on u.token = m.creator_user_token
+order by r.use_count desc
+    limit 100;
+
+--
+-- Same, but simpler...
+--
+select m.token, m.title, r.use_count from (
+  select model_token, count(*) as use_count from tts_results
+  group by model_token
+  order by use_count desc limit 100
+) as r
+  join tts_models as m
+  on m.token = r.model_token;
+
+--
+-- Top 100 models by use count, last 5 days
+-- Limited due to tmux scrollback.
+--
+select
+    m.token,
+    m.text_pipeline_type,
+    m.ietf_language_tag,
+    m.title,
+    u.username,
+    r.use_count,
+    m.created_at,
+    m.user_deleted_at,
+    m.mod_deleted_at
+from (
+    select model_token, count(*) as use_count
+    from tts_results
+    where created_at > ( CURDATE() - INTERVAL 12 HOUR )
+    group by model_token
+) as r
+    join tts_models as m
+    on m.token = r.model_token
+    join users as u
+    on u.token = m.creator_user_token
+order by r.use_count desc
+    limit 100;
+
+
+--
+-- Top models binned by IP address.
+-- Should give us a look at demand independent of model quality / repeated use.
+--
+select
+    m.token,
+    m.title,
+    m.user_ratings_positive_count / m.user_ratings_total_count as rating,
+    m.text_pipeline_type,
+    m.ietf_language_tag,
+    u.username,
+    r.use_count,
+    m.created_at,
+    m.user_deleted_at,
+    m.mod_deleted_at
+from (
+        select model_token, count(*) as use_count
+        from (
+            select model_token, creator_ip_address
+            from tts_results
+            where created_at > (CURDATE() - INTERVAL 12 HOUR)
+            group by model_token, creator_ip_address
+        ) as x
+        group by model_token
+     ) as r
+         join tts_models as m
+              on m.token = r.model_token
+         join users as u
+              on u.token = m.creator_user_token
+         where m.ietf_language_tag IN ('en', 'en-US', 'en-AU', 'en-CA', 'en-GB')
+order by r.use_count desc
+    limit 100;
+
+--
+-- Top 20 ***NEW*** models to share in #announcements
+-- Can be scoped by creator.
+--
+select
+    m.token,
+    m.text_pipeline_type,
+    m.ietf_language_tag,
+    m.creator_set_visibility,
+    m.title,
+    u.username,
+    r.use_count,
+    m.created_at,
+    m.user_deleted_at,
+    m.mod_deleted_at
+from (
+         select model_token, count(*) as use_count
+         from tts_results
+         where created_at > ( CURDATE() - INTERVAL 5 DAY )
+         group by model_token
+     ) as r
+         join tts_models as m
+              on m.token = r.model_token
+         join users as u
+              on u.token = m.creator_user_token
+where m.created_at > ( CURDATE() - INTERVAL 31 DAY )
+  -- AND u.username IN ( 'vegito1089', 'justinjohn0306')
+order by r.use_count desc
+    limit 100;
+
+--
+-- Top 100 models by use count, last 5 days, not spanish, etc.
+-- Limited due to tmux scrollback.
+--
+select
+    m.token,
+    m.text_pipeline_type,
+    m.ietf_language_tag,
+    m.title,
+    u.username,
+    r.use_count,
+    m.created_at,
+    m.user_deleted_at,
+    m.mod_deleted_at
+from (
+    select model_token, count(*) as use_count
+    from tts_results
+    where created_at > ( CURDATE() - INTERVAL 5 DAY )
+    group by model_token
+) as r
+    join tts_models as m
+    on m.token = r.model_token
+    join users as u
+    on u.token = m.creator_user_token
+    where m.ietf_language_tag NOT IN ('es', 'es-419', 'es-ES', 'es-MX', 'pt-BR')
+order by r.use_count desc
+    limit 100;
+
+--
+-- Top 100 models by use count, last 5 days, in specific language.
+-- Limited due to tmux scrollback.
+--
+-- English: ('en', 'en-US', 'en-AU', 'en-CA', 'en-GB')
+-- Spanish: ('es', 'es-419', 'es-AR', 'es-CL', 'es-CO', 'es-ES', 'es-MX', 'es-PE', 'es-US')
+-- Italian: ('it', 'it-CH', 'it-IT')
+-- Portuguese: ('pt', 'pt-BR')
+--
+select
+    m.token,
+    m.title,
+    m.text_pipeline_type,
+    m.ietf_language_tag,
+    u.username,
+    r.use_count,
+    m.created_at,
+    m.user_deleted_at,
+    m.mod_deleted_at
+from (
+         select model_token, count(*) as use_count
+         from tts_results
+         where created_at > ( CURDATE() - INTERVAL 30 DAY )
+         group by model_token
+     ) as r
+         join tts_models as m
+              on m.token = r.model_token
+         join users as u
+              on u.token = m.creator_user_token
+where m.ietf_language_tag IN ('en', 'en-US', 'en-AU', 'en-CA', 'en-GB')
+order by r.use_count desc
+    limit 100;
+
+--
+-- Most popular voices by use count over 5-day window, single language, single user.
+--
+select
+    m.token,
+    m.title,
+    m.ietf_language_tag,
+    u.username,
+    r.use_count,
+    m.created_at,
+    m.user_deleted_at,
+    m.mod_deleted_at
+from (
+    select model_token, count(*) as use_count
+    from tts_results
+    where created_at > ( CURDATE() - INTERVAL 5 DAY )
+    group by model_token
+) as r
+    join tts_models as m
+    on m.token = r.model_token
+    join users as u
+    on u.token = m.creator_user_token
+    where
+        m.ietf_language_tag NOT IN ('es', 'es-419', 'es-ES', 'es-MX', 'pt-BR')
+        AND u.username = 'vegito1089'
+order by r.use_count desc
+    limit 100;
+
+--
+-- Most popular deleted models
+--
+select
+    m.token,
+    m.title,
+    m.ietf_language_tag,
+    u.username,
+    r.use_count,
+    m.created_at,
+    m.user_deleted_at,
+    m.mod_deleted_at
+from (
+    select model_token, count(*) as use_count
+    from tts_results
+    where model_token IN (
+        select token
+        from tts_models
+        where
+            user_deleted_at IS NOT NULL
+            OR mod_deleted_at IS NOT NULL
+    )
+    group by model_token
+) as r
+    join tts_models as m
+    on m.token = r.model_token
+    join users as u
+    on u.token = m.creator_user_token
+order by r.use_count desc
+    limit 100;
+
+--
+-- Most popular deleted models (only by users, not mods)
+-- Mod-deleted models were probably on purpose
+--
+select
+    m.token,
+    m.title,
+    m.ietf_language_tag,
+    u.username,
+    r.use_count,
+    m.created_at,
+    m.user_deleted_at
+from (
+         select model_token, count(*) as use_count
+         from tts_results
+         where model_token IN (
+             select token
+             from tts_models
+             where
+                 user_deleted_at IS NOT NULL
+         )
+         group by model_token
+     ) as r
+         join tts_models as m
+              on m.token = r.model_token
+         join users as u
+              on u.token = m.creator_user_token
+order by r.use_count desc
+    limit 100;
+
+
+--
+-- Models uploaded recently (to check if they use the right text_pipeline_type,
+-- vocoder -todo-, etc.)
+--
+select
+    m.token,
+    m.title,
+    m.text_pipeline_type,
+    m.ietf_language_tag,
+    u.username,
+    r.use_count,
+    m.created_at,
+    m.user_deleted_at,
+    m.mod_deleted_at
+from (
+         select model_token, count(*) as use_count
+         from tts_results
+         where created_at > ( CURDATE() - INTERVAL 5 DAY )
+         group by model_token
+     ) as r
+         join tts_models as m
+              on m.token = r.model_token
+         join users as u
+              on u.token = m.creator_user_token
+order by m.created_at desc
+    limit 100;
+
+-- Histogram of model contributions
+select
+    date(created_at) as created_date,
+    count(*) as model_count
+from tts_models
+group by created_date
+
+-- Histogram of model contributions (within range)
+select
+    date(created_at) as created_date,
+    count(*) as model_count
+from tts_models
+where created_at > (CURDATE() - INTERVAL 5 DAY)
+group by created_date
+
+--
+-- Models deleted recently (voice actor take down)
+--
+select
+    m.token,
+    m.title,
+    m.text_pipeline_type,
+    m.ietf_language_tag,
+    u.username,
+    r.use_count,
+    m.created_at,
+    m.user_deleted_at,
+    m.mod_deleted_at
+from (
+         select model_token, count(*) as use_count
+         from tts_results
+         where created_at > ( CURDATE() - INTERVAL 30 DAY )
+         group by model_token
+     ) as r
+         join tts_models as m
+              on m.token = r.model_token
+         join users as u
+              on u.token = m.creator_user_token
+where m.mod_deleted_at > ( CURDATE() - INTERVAL 14 DAY )
+   or m.user_deleted_at > ( CURDATE() - INTERVAL 14 DAY )
+order by r.use_count desc
+    limit 500;
+
+--
+-- Non-public models
+--
+select
+    m.token,
+    m.title,
+    m.text_pipeline_type,
+    m.ietf_language_tag,
+    m.creator_set_visibility,
+    u.username,
+    r.use_count,
+    m.created_at,
+    m.user_deleted_at,
+    m.mod_deleted_at
+from (
+         select model_token, count(*) as use_count
+         from tts_results
+         where created_at > ( CURDATE() - INTERVAL 30 DAY )
+         group by model_token
+     ) as r
+         left outer join tts_models as m
+              on m.token = r.model_token
+         join users as u
+              on u.token = m.creator_user_token
+where m.creator_set_visibility != 'public'
+order by r.use_count desc
+    limit 500;
+

_sql/analytics/tts/tts_results.sql

@@ -0,0 +1,149 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+--
+-- Usage count in the past 24 hours
+--
+select count(*) as use_count
+from tts_results
+where created_at > ( CURDATE() - INTERVAL 1 DAY )
+
+--
+-- Usage count in the past week
+--
+select count(*) as use_count
+from tts_results
+where created_at > ( CURDATE() - INTERVAL 7 DAY )
+
+-- On-prem worker mix
+select count(*) as use_count
+from tts_results
+where created_at > ( CURDATE() - INTERVAL 1 MINUTE )
+and is_generated_on_prem IS TRUE
+
+-- Calculate a percentage of on-prem worker capacity
+select count(*) as on_prem_count
+from (
+  select is_generated_on_prem
+  from tts_results
+  order by id desc limit 1000
+) as sample
+where sample.is_generated_on_prem IS TRUE;
+
+-- Histogram of days of content per day
+select
+    date(created_at) as created_date,
+    sum(duration_millis) / 1000 / 60 / 60 / 24 as days
+from tts_results
+group by created_date
+
+-- Histogram of days of content per day (range)
+select
+    date(created_at) as created_date,
+    sum(duration_millis) / 1000 / 60 / 60 / 24 as days
+from tts_results
+where date(created_at) >= "2023-01-01"
+group by created_date
+
+-- Histogram of days of content per day (range)
+-- Maybe non-tablescan?
+select
+    date(created_at) as created_date,
+    sum(duration_millis) / 1000 / 60 / 60 / 24 as days
+from tts_results
+where created_at > (CURDATE() - INTERVAL 30 DAY)
+group by created_date
+
+-- Find TTS results for a single model
+select count(*) from tts_results
+where model_token = 'TM:ztt5s1be5tq6';
+
+-- Find TTS results for a single model that are not deleted
+select count(*) from tts_results
+where model_token = 'TM:ztt5s1be5tq6'
+and mod_deleted_at is NULL;
+
+-- Sample results for a single model
+select
+    TRIM(REPLACE(SUBSTRING(raw_inference_text, 1, 50), '\n', ''))
+from tts_results
+where model_token='TM:yt4gfbkngsjj'
+order by id desc
+limit 150;
+
+-- Search for results by eminem
+select
+    token,
+    created_at,
+    raw_inference_text
+from  tts_results
+where
+    model_token IN ('TM:27fj0gsh11pd', 'TM:8h9bfjgabeer', 'TM:pdf9c1anbdjq')
+    AND raw_inference_text LIKE "%rave%"
+  AND raw_inference_text LIKE "%sound%"
+  AND created_at > ( CURDATE() - INTERVAL 5 MONTH )
+ORDER BY created_at DESC
+
+
+
+--- Search for particular text by a particular model
+SELECT
+    raw_inference_text,
+    maybe_creator_user_token,
+    created_at
+FROM
+    tts_results
+WHERE
+    model_token = 'TM:ztt5s1be5tq6'
+    AND raw_inference_text LIKE '%fakeyou%'
+
+--- Search for 101soundboards.com using us
+SELECT
+    r.raw_inference_text,
+    r.maybe_creator_user_token,
+    u.username,
+    u.ip_address_creation,
+    r.creator_ip_address,
+    r.created_at
+FROM
+    tts_results as r
+LEFT OUTER JOIN
+    users as u
+ON
+    r.maybe_creator_user_token = u.token
+WHERE
+    r.raw_inference_text LIKE '%bananas%'
+    AND r.created_at > ( CURDATE() - INTERVAL 1 HOUR )
+
+
+--- Search for 101soundboards.com using us
+SELECT
+    r.maybe_creator_user_token, r.creator_ip_address, r.raw_inference_text
+FROM
+    tts_results as r
+        LEFT OUTER JOIN
+    users as u
+    ON
+            r.maybe_creator_user_token = u.token
+WHERE
+        r.raw_inference_text LIKE '%pickle%'
+  AND r.created_at > ( CURDATE() - INTERVAL 1 HOUR )
+
+
+---
+-- WTF WHY IS THIS SLOW???
+SELECT
+    res.id,
+    model.title,
+    res.raw_inference_text,
+    res.maybe_creator_user_token
+FROM
+    tts_results AS res
+JOIN
+    tts_models as model
+ON
+    res.model_token = model.token
+ORDER BY res.id
+LIMIT 10;
+

_sql/analytics/users/subscribers/loyal_user_cohorts_mom.sql

@@ -0,0 +1,56 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Example query for date ranges:
+select count(*)
+from users
+where created_at BETWEEN '2023-07-01 00:00:00' AND '2023-07-30 00:00:00';
+
+--
+
+-- Month by month subscriber usages.
+SELECT
+    username,
+    count(*) as total_use_count
+FROM (
+         SELECT
+             u.username as username,
+             j.token as token
+         FROM users AS u
+                  JOIN tts_inference_jobs AS j
+                       ON u.token = j.maybe_creator_user_token
+         WHERE u.token IN (
+             SELECT DISTINCT user_token
+             FROM user_subscriptions
+         )
+           AND j.created_at BETWEEN '2022-12-01 00:00:00' AND '2023-01-01 00:00:00'
+         UNION
+         SELECT
+             u.username as username,
+             j.token as token
+         FROM users AS u
+             JOIN generic_inference_jobs AS j
+         ON u.token = j.maybe_creator_user_token
+         WHERE u.token IN (
+             SELECT DISTINCT user_token
+             FROM user_subscriptions
+             )
+           AND j.created_at BETWEEN '2022-12-01 00:00:00' AND '2023-01-01 00:00:00'
+         UNION
+         SELECT
+             u.username as username,
+             j.token as token
+         FROM users AS u
+             JOIN w2l_inference_jobs AS j
+         ON u.token = j.maybe_creator_user_token
+         WHERE u.token IN (
+             SELECT DISTINCT user_token
+             FROM user_subscriptions
+             )
+           AND j.created_at BETWEEN '2022-12-01 00:00:00' AND '2023-01-01 00:00:00'
+     ) as t
+GROUP BY username
+ORDER BY total_use_count desc;
+
+

_sql/analytics/users/subscribers/paying_user_interests.sql

@@ -0,0 +1,161 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- TODO: Do an analysis of voice binned by # of paying customers that use it.
+
+-- The most popular models amongst paying subscribers.
+-- This should give us insight as to which models subscribers pay us for.
+SELECT *
+FROM (
+         SELECT
+             t.token,
+             t.title,
+             count(*) as use_count,
+             t.text_pipeline_type,
+             t.ietf_language_tag,
+             t.created_at,
+             t.user_deleted_at,
+             t.mod_deleted_at
+         FROM
+             tts_inference_jobs as j
+                 JOIN
+             tts_models as t
+             ON
+                     t.token = j.model_token
+         WHERE j.maybe_creator_user_token IN
+               (
+                   select distinct user_token
+                   from user_subscriptions
+               )
+           AND
+                 j.created_at > ( CURDATE() - INTERVAL 1 DAY )
+         GROUP BY t.token
+         ORDER BY use_count DESC
+     ) as x
+     WHERE
+        x.use_count > 10;
+
+
+-- The first TTS voice a paying customer uses
+-- TODO: List *all* of the voices a paying customer uses before they subscribe, not just the first.
+-- select first row of partially aggregate query: https://stackoverflow.com/a/73157541
+SELECT *
+FROM (
+         SELECT
+             t.token,
+             t.title,
+             count(*) as use_count,
+             t.text_pipeline_type,
+             t.ietf_language_tag,
+             t.created_at,
+             t.user_deleted_at,
+             t.mod_deleted_at
+         FROM tts_models AS t
+                  JOIN
+              (
+                  SELECT
+                      model_token
+                  FROM (
+                           SELECT
+                               model_token,
+                               maybe_creator_user_token,
+                               row_number() over(partition by maybe_creator_user_token order by id asc) as rownum
+                           FROM
+                               tts_inference_jobs
+                           WHERE maybe_creator_user_token IN
+                                 (
+                                     select distinct user_token
+                                     from user_subscriptions
+                                     where created_at > (CURDATE() - INTERVAL 2 DAY)
+                                 )
+                       ) AS x
+                  WHERE rownum = 1
+                  ORDER BY model_token desc
+              ) AS y
+              ON t.token = y.model_token
+         GROUP BY t.token
+     ) AS z
+ORDER BY use_count desc
+
+
+-- The first VC voice a paying customer uses
+-- TODO: List *all* of the voices a paying customer uses before they subscribe, not just the first.
+-- select first row of partially aggregate query: https://stackoverflow.com/a/73157541
+SELECT *
+FROM (
+         SELECT
+             m.token,
+             m.title,
+             count(*) as use_count,
+             m.created_at,
+             m.user_deleted_at,
+             m.mod_deleted_at
+         FROM voice_conversion_models AS m
+                  JOIN
+              (
+                  SELECT
+                      model_token
+                  FROM (
+                           SELECT
+                               maybe_model_token as model_token,
+                               maybe_creator_user_token,
+                               row_number() over(partition by maybe_creator_user_token order by id asc) as rownum
+                           FROM
+                               generic_inference_jobs
+                           WHERE maybe_model_token IS NOT NULL
+                           AND maybe_creator_user_token IN
+                                 (
+                                     select distinct user_token
+                                     from user_subscriptions
+                                     where created_at > (CURDATE() - INTERVAL 2 DAY)
+                                 )
+                       ) AS x
+                  WHERE rownum = 1
+                  ORDER BY model_token desc
+              ) AS y
+              ON m.token = y.model_token
+         GROUP BY m.token
+     ) AS z
+ORDER BY use_count desc
+
+
+-- The last voice a paying customer uses
+-- select first row of partially aggregate query: https://stackoverflow.com/a/73157541
+SELECT *
+FROM (
+    SELECT
+      t.token,
+      t.title,
+      count(*) as use_count,
+      t.text_pipeline_type,
+      t.ietf_language_tag,
+      t.created_at,
+      t.user_deleted_at,
+      t.mod_deleted_at
+    FROM tts_models AS t
+    JOIN
+        (
+            SELECT
+                model_token
+            FROM (
+                SELECT
+                    model_token,
+                    maybe_creator_user_token,
+                    row_number() over(partition by maybe_creator_user_token order by id desc) as rownum
+                FROM
+                    tts_inference_jobs
+                WHERE maybe_creator_user_token IN
+                      (
+                          select distinct user_token
+                          from user_subscriptions
+                      )
+            ) AS x
+            WHERE rownum = 1
+            ORDER BY model_token desc
+          ) AS y
+    ON t.token = y.model_token
+    GROUP BY t.token
+) AS z
+ORDER BY use_count desc
+

_sql/analytics/users/subscribers/paying_user_retention.sql

@@ -0,0 +1,170 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+
+-- Number of paid users (inaccurate)
+select count(*) from users where maybe_stripe_customer_id IS NOT NULL;
+
+-- Number of paid users (inaccurate)
+select count(*) from user_subscriptions;
+
+-- Number of paid users
+-- NB: "active" status users are current, treating "past_due" as active too (for now)
+select count(*) from user_subscriptions where maybe_stripe_subscription_status NOT IN ("canceled", "incomplete", "incomplete_expired");
+
+-- ==================== TTS + VC + W2L ====================
+
+-- Make sure the time intervals are the same!
+SELECT
+    username,
+    count(*) as total_use_count
+FROM (
+   SELECT
+        u.username as username,
+        j.token as token
+    FROM users AS u
+    JOIN tts_inference_jobs AS j
+        ON u.token = j.maybe_creator_user_token
+        WHERE u.token IN (
+            SELECT DISTINCT user_token
+            FROM user_subscriptions
+        )
+        AND j.created_at > NOW() - INTERVAL 31 DAY
+   UNION
+        SELECT
+            u.username as username,
+            j.token as token
+        FROM users AS u
+            JOIN generic_inference_jobs AS j
+        ON u.token = j.maybe_creator_user_token
+        WHERE u.token IN (
+            SELECT DISTINCT user_token
+            FROM user_subscriptions
+            )
+          AND j.created_at > NOW() - INTERVAL 31 DAY
+    UNION
+        SELECT
+            u.username as username,
+            j.token as token
+        FROM users AS u
+             JOIN w2l_inference_jobs AS j
+        ON u.token = j.maybe_creator_user_token
+        WHERE u.token IN (
+            SELECT DISTINCT user_token
+            FROM user_subscriptions
+            )
+        AND j.created_at > NOW() - INTERVAL 31 DAY
+) as t
+GROUP BY username
+ORDER BY total_use_count desc
+
+-- ==================== TEXT TO SPEECH====================
+
+-- All paid subscribers generating results in the last week
+select u.username, count(*) as use_count
+from users as u
+join tts_inference_jobs as j
+on u.token = j.maybe_creator_user_token
+WHERE u.token IN (
+    select distinct user_token
+    from user_subscriptions
+    -- where maybe_stripe_subscription_status NOT IN ("canceled", "incomplete", "incomplete_expired")
+)
+AND j.created_at > NOW() - interval 7 day
+GROUP BY u.username
+ORDER BY use_count desc
+
+-- Paid subscribers generating results in the last month (with # threshold)
+select username, use_count
+FROM (
+    select u.username, count(*) as use_count
+    from users as u
+             join tts_inference_jobs as j
+                  on u.token = j.maybe_creator_user_token
+    WHERE u.token IN (
+        select distinct user_token
+        from user_subscriptions
+        where maybe_stripe_subscription_status NOT IN ("canceled", "incomplete", "incomplete_expired")
+    )
+    AND j.created_at > NOW() - interval 31 day
+    GROUP BY u.username
+    ORDER BY use_count desc
+) as t
+WHERE use_count > 10
+
+-- Paid users generating results in the last month
+select u.username, count(*) as use_count
+from users as u
+      join tts_inference_jobs as j
+           on u.token = j.maybe_creator_user_token
+WHERE u.token IN (
+     select distinct user_token
+     from user_subscriptions
+     where maybe_stripe_subscription_status NOT IN ("canceled", "incomplete", "incomplete_expired")
+)
+AND j.created_at > NOW() - interval 1 day
+GROUP BY u.username
+ORDER BY use_count desc
+
+-- ==================== WAV2LIP ====================
+
+select u.username, count(*) as use_count
+from users as u
+         join w2l_inference_jobs as j
+              on u.token = j.maybe_creator_user_token
+WHERE u.token IN (
+    select distinct user_token
+    from user_subscriptions
+    where maybe_stripe_subscription_status NOT IN ("canceled", "incomplete", "incomplete_expired")
+)
+  AND j.created_at > NOW() - interval 31 day
+GROUP BY u.username
+ORDER BY use_count desc
+
+select username, use_count
+FROM (
+         select u.username, count(*) as use_count
+         from users as u
+                  join w2l_inference_jobs as j
+                       on u.token = j.maybe_creator_user_token
+         WHERE u.token IN (
+             select distinct user_token
+             from user_subscriptions
+             where maybe_stripe_subscription_status NOT IN ("canceled", "incomplete", "incomplete_expired")
+         )
+           AND j.created_at > NOW() - interval 31 day
+         GROUP BY u.username
+         ORDER BY use_count desc
+     ) as t
+WHERE use_count >=5;
+
+-- ==================== TTS MODELS ====================
+
+select u.username, count(*) as use_count
+from users as u
+         join tts_model_upload_jobs as j
+              on u.token = j.creator_user_token
+WHERE u.token IN (
+    select distinct user_token
+    from user_subscriptions
+    where maybe_stripe_subscription_status NOT IN ("canceled", "incomplete", "incomplete_expired")
+)
+  AND j.created_at > NOW() - interval 1 day
+GROUP BY u.username
+ORDER BY use_count desc
+
+-- ==================== W2L TEMPLATES ====================
+
+select u.username, count(*) as use_count
+from users as u
+         join w2l_template_upload_jobs as j
+              on u.token = j.creator_user_token
+WHERE u.token IN (
+    select distinct user_token
+    from user_subscriptions
+    where maybe_stripe_subscription_status NOT IN ("canceled", "incomplete", "incomplete_expired")
+)
+  AND j.created_at > NOW() - interval 1 day
+GROUP BY u.username
+ORDER BY use_count desc

_sql/analytics/users/top_users_by_usage.sql

@@ -0,0 +1,53 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+SELECT
+    u.username as username,
+    j.token as token
+FROM users AS u
+JOIN tts_inference_jobs AS j
+ON u.token = j.maybe_creator_user_token
+  AND j.created_at > NOW() - INTERVAL 31 DAY
+
+
+SELECT
+    t.maybe_creator_user_token as user_token,
+    u.username,
+    count(*) as usage_count,
+    u.created_at
+FROM tts_results as t
+JOIN users as u
+ON u.token = t.maybe_creator_user_token
+WHERE t.maybe_creator_user_token IS NOT NULL
+GROUP BY t.maybe_creator_user_token
+ORDER BY usage_count DESC
+LIMIT 50000;
+
+
+SELECT
+    v.maybe_creator_user_token as user_token,
+    u.username,
+    count(*) as usage_count
+FROM voice_conversion_results as v
+JOIN users as u
+ON u.token = v.maybe_creator_user_token
+WHERE v.maybe_creator_user_token IS NOT NULL
+GROUP BY v.maybe_creator_user_token
+ORDER BY usage_count DESC
+LIMIT 5000;
+
+
+SELECT
+    w.maybe_creator_user_token as user_token,
+    u.username,
+    count(*) as usage_count
+FROM w2l_results as w
+         JOIN users as u
+              ON u.token = w.maybe_creator_user_token
+WHERE w.maybe_creator_user_token IS NOT NULL
+GROUP BY w.maybe_creator_user_token
+ORDER BY usage_count DESC
+LIMIT 5000;
+
+

_sql/analytics/users/user_registrations.sql

@@ -0,0 +1,19 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- All users
+select count(*) from users;
+
+-- Users registered in the last year
+select count(*)
+from users
+where created_at > ( CURDATE() - INTERVAL 365 DAY )
+
+-- Histogram of user signups
+select
+    date(created_at) as created_date,
+    count(*) as registered
+from users
+where date(created_at) >= "2023-01-01"
+group by created_date