windows fixes and performance fixes

unfortunately cpal doesn't support wasm/webaudio input devices https://github.com/RustAudio/cpal/issues/412

.dockerignore

@@ -1,11 +1,9 @@
 *secret*sh
 *~
-.dockerignore
 .env
 .env-secrets
 .git
 .idea/
-Dockerfile
 __pycache__/
 _ignored_schema.rs
 models/Wav2Lip/python/

.env

@@ -1,27 +1,16 @@
-SERVER_ENVIRONMENT=development
-
 DATABASE_URL=mysql://storyteller:password@localhost/storyteller
 #DATABASE_URL=mysql://root:root@localhost/storyteller
 #DATABASE_URL=mysql://root@localhost/storyteller
 
-# Diesel-cli reads this for the migration directory to use.
-# We've renamed it so the migrations are sorted near the top of the project directory tree
-# (since they kind of serve as documentation)
-MIGRATION_DIRECTORY=_sql/migrations/
-
 # SQLX normally requires a live database to reflect upon at build time.
 # This setting turns that on and requires a `sql-data.json` cache file instead.
 # See https://github.com/launchbadge/sqlx/tree/master/sqlx-cli#readme
 #SQLX_OFFLINE=true
 
-# NB(bt, 2023-03-20):
-#  These are used in storyteller-web, image-generation-service, and websocket-gateway.
-#  I'm moving them to env configs that live under storyteller-web/.env.
-#  This will break the other services for now.
-# # We can't serve HTTPS locally without self-signed certificate nonsense.
-# COOKIE_SECURE=false
-# COOKIE_DOMAIN=.jungle.horse
-# WEBSITE_HOMEPAGE_REDIRECT=http://jungle.horse
+# We can't serve HTTPS locally without self-signed certificate nonsense.
+COOKIE_SECURE=false
+COOKIE_DOMAIN=.jungle.horse
+WEBSITE_HOMEPAGE_REDIRECT=http://jungle.horse
 
 # Bucket config (non-secret)
 REGION_NAME=us-east1

.gitattributes

@@ -1,5 +0,0 @@
-# Ignore certain code for Github language statistics
-# https://github.com/github/linguist/blob/master/docs/overrides.md
-old_code/trumpet-trumped.com/dictionary/cmudict-0.7b linguist-vendored
-old_code/voder-vocodes-1.0/frontends/admin/node_modules linguist-vendored
-windows/sqlite linguist-vendored

.github/workflows/docker-publish-cpu.yml

@@ -1,113 +0,0 @@
-# Adapted from
-# https://github.com/actions/starter-workflows/ci/docker-publish.yml
-# If using base images, don't forget to add READ access, eg. from a package settings page like:
-# https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access
-name: "[CPU] Build GHCR Docker artifact"
-on:
-  push:
-    branches:
-      # NB: Default-branch doesn't work, despite Github's documentation.
-      #- $default-branch
-      - master
-env:
-  IMAGE_NAME: storyteller-rust
-
-jobs:
-  # Push image to GitHub Packages.
-  # See also https://docs.docker.com/docker-hub/builds/
-  # See also https://depot.dev/blog/docker-layer-caching-in-github-actions
-  # See also https://evilmartians.com/chronicles/build-images-on-github-actions-with-docker-layer-caching
-  build-docker-image:
-    name: "Build Docker Image"
-    runs-on: ubuntu-22.04 # runs-on: ubuntu-latest
-    if: github.event_name == 'push'
-    permissions:
-      packages: write
-      contents: read
-
-    steps:
-      - uses: actions/checkout@v3
-
-      # - uses: docker/setup-buildx-action@v1
-      #   name: Set up Docker Buildx
-
-      - uses: docker/login-action@v1
-        name: Login to GitHub Container Registry
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }} # username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Calculate short SHA
-        id: vars
-        # echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-        run: |
-          echo "sha_short=$(echo ${{ github.sha }} | cut -c1-12)" >> $GITHUB_OUTPUT
-
-      - name: Emit short SHA
-        run: |
-          echo "Full SHA: ${{ github.sha }}"
-          echo "Short SHA: ${{ steps.vars.outputs.sha_short }}"
-
-      #- uses: actions/cache@v2
-      #  name: Cache Docker layers
-      #  with:
-      #    path: /tmp/.buildx-cache
-      #    key: ${{ runner.os }}-buildx-${{ github.sha }}
-      #    restore-keys: |
-      #      ${{ runner.os }}-buildx-
-
-      # Docs for this action are at https://github.com/marketplace/actions/build-docker-images-using-cache
-      - uses: whoan/docker-build-with-cache-action@v6
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }} # username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-          image_name: storytold/storyteller-rust
-          dockerfile: service_cpu.Dockerfile
-          # NB: The `git_push_tag: true` option does not work as expected, so we put the sha in `image_tag`:
-          # https://github.com/whoan/docker-build-with-cache-action/issues/72
-          image_tag: latest,${{ github.sha }},${{ steps.vars.outputs.sha_short }}
-          # NB: Inject the Git SHA into the container so the servers can emit their own version:
-          build_extra_args: "--build-arg=GIT_SHA=${{ github.sha }}"
-
-      # - name: Build image
-      #   run: |
-      #     SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
-      #     docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" --build-arg GIT_SHA=${SHORT_SHA}
-
-      # - name: Push image to GitHub Container Registry
-      #   run: |
-      #     IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
-
-      #     # Change all uppercase to lowercase
-      #     IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
-
-      #     # Strip git ref prefix from version
-      #     VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-
-      #     # Strip "v" prefix from tag name
-      #     [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
-
-      #     # Use Docker `latest` tag convention
-      #     [ "$VERSION" == "$default-branch" ] && VERSION=latest
-
-      #     echo IMAGE_ID=$IMAGE_ID
-      #     echo VERSION=$VERSION
-
-      #     docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
-      #     docker push $IMAGE_ID:$VERSION
-
-      #     SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
-
-      #     docker tag $IMAGE_NAME $IMAGE_ID:$SHORT_SHA
-      #     docker push $IMAGE_ID:$SHORT_SHA
-
-      - name: Cleanup / Debug
-        # This job will run even on failures to help with debugging.
-        if: always()
-        run: |
-          echo "Report at final github action step:"
-          echo "Disk usage:"
-          pwd
-          du -hsc * | sort -hr

.github/workflows/docker-publish-gpu.yml

@@ -1,113 +0,0 @@
-# Adapted from
-# https://github.com/actions/starter-workflows/ci/docker-publish.yml
-# If using base images, don't forget to add READ access, eg. from a package settings page like:
-# https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access
-name: "[GPU] Build GHCR Docker artifact"
-on:
-  push:
-    branches:
-      # NB: Default-branch doesn't work, despite Github's documentation.
-      #- $default-branch
-      - master
-env:
-  IMAGE_NAME: storyteller-rust-gpu
-
-jobs:
-  # Push image to GitHub Packages.
-  # See also https://docs.docker.com/docker-hub/builds/
-  # See also https://depot.dev/blog/docker-layer-caching-in-github-actions
-  # See also https://evilmartians.com/chronicles/build-images-on-github-actions-with-docker-layer-caching
-  build-docker-image:
-    name: "Build Docker Image"
-    runs-on: ubuntu-22.04 # runs-on: ubuntu-latest
-    if: github.event_name == 'push'
-    permissions:
-      packages: write
-      contents: read
-
-    steps:
-      - uses: actions/checkout@v3
-
-      # - uses: docker/setup-buildx-action@v1
-      #   name: Set up Docker Buildx
-
-      - uses: docker/login-action@v1
-        name: Login to GitHub Container Registry
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }} # username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Calculate short SHA
-        id: vars
-        # echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-        run: |
-          echo "sha_short=$(echo ${{ github.sha }} | cut -c1-12)" >> $GITHUB_OUTPUT
-
-      - name: Emit short SHA
-        run: |
-          echo "Full SHA: ${{ github.sha }}"
-          echo "Short SHA: ${{ steps.vars.outputs.sha_short }}"
-
-      #- uses: actions/cache@v2
-      #  name: Cache Docker layers
-      #  with:
-      #    path: /tmp/.buildx-cache
-      #    key: ${{ runner.os }}-buildx-${{ github.sha }}
-      #    restore-keys: |
-      #      ${{ runner.os }}-buildx-
-
-      # Docs for this action are at https://github.com/marketplace/actions/build-docker-images-using-cache
-      - uses: whoan/docker-build-with-cache-action@v6
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }} # username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-          image_name: storytold/storyteller-rust-gpu
-          dockerfile: service_gpu.Dockerfile
-          # NB: The `git_push_tag: true` option does not work as expected, so we put the sha in `image_tag`:
-          # https://github.com/whoan/docker-build-with-cache-action/issues/72
-          image_tag: latest,${{ github.sha }},${{ steps.vars.outputs.sha_short }}
-          # NB: Inject the Git SHA into the container so the servers can emit their own version:
-          build_extra_args: "--build-arg=GIT_SHA=${{ github.sha }}"
-
-      # - name: Build image
-      #   run: |
-      #     SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
-      #     docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" --build-arg GIT_SHA=${SHORT_SHA}
-
-      # - name: Push image to GitHub Container Registry
-      #   run: |
-      #     IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
-
-      #     # Change all uppercase to lowercase
-      #     IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
-
-      #     # Strip git ref prefix from version
-      #     VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-
-      #     # Strip "v" prefix from tag name
-      #     [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
-
-      #     # Use Docker `latest` tag convention
-      #     [ "$VERSION" == "$default-branch" ] && VERSION=latest
-
-      #     echo IMAGE_ID=$IMAGE_ID
-      #     echo VERSION=$VERSION
-
-      #     docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
-      #     docker push $IMAGE_ID:$VERSION
-
-      #     SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
-
-      #     docker tag $IMAGE_NAME $IMAGE_ID:$SHORT_SHA
-      #     docker push $IMAGE_ID:$SHORT_SHA
-
-      - name: Cleanup / Debug
-        # This job will run even on failures to help with debugging.
-        if: always()
-        run: |
-          echo "Report at final github action step:"
-          echo "Disk usage:"
-          pwd
-          du -hsc * | sort -hr

.github/workflows/docker-publish.yml

@@ -0,0 +1,67 @@
+# Adapted from
+# https://github.com/actions/starter-workflows/ci/docker-publish.yml
+# If using base images, don't forget to add READ access, eg. from a package settings page like:
+# https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access
+name: Publish Docker Container to GHCR
+on:
+  push:
+    branches:
+      # NB: Default-branch doesn't work, despite Github's documentation.
+      #- $default-branch
+      - master
+env:
+  IMAGE_NAME: storyteller-rust
+
+jobs:
+  # Push image to GitHub Packages.
+  # See also https://docs.docker.com/docker-hub/builds/
+  push:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          #username: ${{ github.repository_owner }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }} 
+
+      - name: Build image
+        run: |
+          SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
+          docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" --build-arg GIT_SHA=${SHORT_SHA}
+
+      - name: Push image to GitHub Container Registry
+        run: |
+          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+
+          # Change all uppercase to lowercase
+          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+
+          # Strip git ref prefix from version
+          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+
+          # Strip "v" prefix from tag name
+          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+
+          # Use Docker `latest` tag convention
+          [ "$VERSION" == "$default-branch" ] && VERSION=latest
+
+          echo IMAGE_ID=$IMAGE_ID
+          echo VERSION=$VERSION
+
+          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION
+
+          SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
+
+          docker tag $IMAGE_NAME $IMAGE_ID:$SHORT_SHA
+          docker push $IMAGE_ID:$SHORT_SHA
+

.github/workflows_inactive/ci.yml

@@ -1,52 +0,0 @@
-name: CI
-on: [push, pull_request]
-env: 
-  MYSQL_URL: mysql://root:root@localhost/storyteller
-  DATABASE_URL: mysql://root:root@localhost/storyteller
-  REDIS_0_URL: redis://redisd@localhost/0
-  REDIS_1_URL: redis://redisd@localhost/1
-  DEBIAN_FRONTEND: noninteractive
-
-jobs:
-  setup_build_test:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-      - name: Cache cargo registry
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
-      - name: Setup mysql
-        run: |
-          sudo systemctl start mysql
-          sudo apt-get update
-          sudo apt-get -y install mysql-client libmysqlclient-dev
-          mysql -e "create user 'storyteller'@'localhost' IDENTIFIED BY 'password';" -uroot -proot 
-          mysql -e "create database storyteller; grant all on storyteller.* to 'storyteller'@'localhost';" -uroot -proot
-      - name: Setup Redis
-        run: |
-          sudo apt-get -y install redis-server
-          sudo service redis-server start
-      - name: Setup Rust
-        run: |
-          rustup set profile minimal
-          rustup show
-          rustup install 1.66
-          rustup default 1.66
-          rustup show
-          cargo install sqlx-cli --features openssl-vendored
-          cargo install diesel_cli
-      - name: Run database migrations
-        run: |
-          # nothing to see here ;)
-          set +e +o pipefail
-          diesel migration run
-          set -e -o pipefail
-      - name: Build 
-        run: cargo build --release
-      - name: Test
-        run: cargo test --release

.gitignore

@@ -1,59 +1,22 @@
-# Secrets are BAD!
+*.swo
+*.swp
 *secret*sh
+*~
 .env
-.env-aichatbot-secrets
 .env-secrets
+.idea/
+__pycache__/
+_ignored_schema.rs
 db.secrets
 db_secrets.toml
+models/Wav2Lip/python/
+models/tacotron2/python/
 production_database_secret_connect.sh
 production_database_secret_query.sh
 production_database_voice_clone_report_download.sh
 secrets.toml
 secrets.yaml
-_develop/tools/.secrets
-_develop/tools/db_secrets.toml
-twitch_secrets.toml
-crates/service/web/storyteller_web/config/storyteller-web.development-secrets.env
-
-# New env secrets (per-app)
-*-secrets.env
-
-# Rust build directory
 target/
-crates/lib/database_queries/target/
-crates/lib/sqlite_queries/target/
-
-# Vim noise
-*.swo
-*.swp
-*~
-
-# IntelliJ/CLion noise
-.idea/
-
-# Python noise
-__pycache__/
-
-# We use diesel for DB migrations only. We don't care about its codgen.
-_ignored_schema.rs
-
-# AI Chat Sidecar has its own schema managed by diesel, but we don't care for diesel codegen.
-crates/programs/aichatbot-sidecar/_ignored_schema.rs
-
-# AI Chat sidecar's SQlite database shouldn't be checked in
-crates/programs/aichatbot-sidecar/database.db
-
-# Runtime data for the AI chatbot
-runtime_data/
-
-# Old Python virtual environments
-models/Wav2Lip/python/
-models/tacotron2/python/
-
-# Stuff I want to keep around, but not commit
-scraps/
-
-# Windows build files
-# These should be built by the platform
-windows/sqlite/sqlite3.exp
-windows/sqlite/sqlite3.lib
+tools/.secrets
+tools/db_secrets.toml
+twitch_secrets.toml

.idea/dictionaries/bt.xml

@@ -1,42 +1,18 @@
 <component name="ProjectDictionaryState">
+  <!--
+  Despite spellcheck dictionaries being created per-user, IntelliJ loads all of them, per:
+  https://stackoverflow.com/a/28151692
+  -->
   <dictionary name="bt">
     <words>
       <w>arpabet</w>
       <w>backfill</w>
-      <w>cashapp</w>
       <w>cheermote</w>
-      <w>crockford</w>
-      <w>culted</w>
-      <w>dalle</w>
-      <w>demuxer</w>
-      <w>downvote</w>
       <w>fakeyou</w>
-      <w>ffprobe</w>
-      <w>greenlight</w>
-      <w>greenlighting</w>
-      <w>greenlit</w>
-      <w>hanashi</w>
       <w>hifigan</w>
-      <w>hparams</w>
-      <w>huggingface</w>
-      <w>pathbuf</w>
-      <w>replayable</w>
-      <w>retryably</w>
       <w>scrollback</w>
-      <w>softvc</w>
-      <w>sql's</w>
-      <w>symphonia</w>
-      <w>tacotron</w>
-      <w>techcrunch</w>
-      <w>transcoded</w>
-      <w>transcodings</w>
-      <w>unpermitted</w>
-      <w>upserting</w>
-      <w>vits</w>
-      <w>vocoded</w>
       <w>vocoder</w>
       <w>vocoders</w>
-      <w>vocodes</w>
     </words>
   </dictionary>
 </component>

Cargo.toml

@@ -1,73 +1,29 @@
 [workspace]
-resolver="2"
 
 members = [
-  # Types: common strongly-typed tokens, ids, payloads, etc.
-  # Also common cloud buckets and other "strongly typed" configs.
-  "crates/_types/buckets",
-  "crates/_types/enums",
-  "crates/_types/reusable_types", # TODO(bt, 2022-12-21): Deprecate
-  "crates/_types/tokens",
-
-  # CLI tools for operations, etc.
-  "crates/cli/database_backup",
-  "crates/cli/database_cleanup",
-  "crates/cli/scrape_test",
-
   # Shared libraries
-  "crates/lib/actix_helpers",
-  "crates/lib/api_clients/cloud_storage",
-  "crates/lib/api_clients/fakeyou_client",
-  "crates/lib/api_clients/google_drive_common",
-  "crates/lib/api_clients/twitch_common",
-  "crates/lib/bootstrap",
-  "crates/lib/caching/memory_caching",
-  "crates/lib/caching/redis_caching",
-  "crates/lib/concurrency",
-  "crates/lib/crockford",
-  "crates/lib/datetimes",
-  "crates/lib/deprecated/config", # TODO(bt, 2023-02-13): Deprecate or repurpose
-  "crates/lib/deprecated/container_common", # TODO(bt, 2023-02-13): Pull apart and rehome pieces
-  "crates/lib/deprecated/http_server_common", # TODO(bt, 2023-03-20): Move useful pieces to actix_helpers
-  "crates/lib/envvar",
-  "crates/lib/errors",
-  "crates/lib/files/filesys",
-  "crates/lib/files/hashing",
-  "crates/lib/files/media",
-  "crates/lib/files/mimetypes",
-  "crates/lib/idempotency",
+  "crates/lib/config",
+  "crates/lib/container_common",
+  "crates/lib/database_queries",
+  "crates/lib/google_drive_common",
+  "crates/lib/http_server_common",
   "crates/lib/jobs_common",
-  "crates/lib/mysql_queries",
+  "crates/lib/memory_caching",
   "crates/lib/redis_common",
-  "crates/lib/server_environment",
-  "crates/lib/sqlite_queries",
-  "crates/lib/sqlx_mysql_helpers",
-  "crates/lib/subprocess_common",
+  "crates/lib/reusable_types",
+  "crates/lib/storage_buckets_common",
   "crates/lib/tts_common",
-  "crates/lib/url_config",
+  "crates/lib/twitch_common",
   "crates/lib/user_input_common",
-  "crates/lib/web_scrapers",
 
-  # Programs: Desktop apps
-  "crates/programs/aichatbot-sidecar",
-  "crates/programs/bevy-experiments",
-
-  # Services: API servers, jobs, workers, etc.
-  "crates/service/job/analytics_job",
-  "crates/service/job/download_job",
-  "crates/service/job/inference_job",
-  "crates/service/job/tts_download_job",
-  "crates/service/job/tts_inference_job",
-  "crates/service/job/w2l_download_job",
-  "crates/service/job/w2l_inference_job",
-  "crates/service/other/twitch_pubsub_subscriber",
-  "crates/service/web/dummy_service",
-  "crates/service/web/image_generation_service",
-  "crates/service/web/storyteller_web",
-  "crates/service/web/websocket_gateway",
-
-  # Service Plugins: complete subsystems that can be plugged into services (users, billing, etc.)
-  "crates/service/plugins/billing",
-  "crates/service/plugins/inference_workloads",
-  "crates/service/plugins/users",
+  # Application 
+  "crates/service/generic_download_job",
+  "crates/service/storyteller_web",
+  "crates/service/tts_download_job",
+  "crates/service/tts_inference_job",
+  "crates/service/twitch_pubsub_subscriber",
+  "crates/service/w2l_download_job",
+  "crates/service/w2l_inference_job",
+  "crates/service/websocket_gateway",
 ]
+

Dockerfile

@@ -0,0 +1,187 @@
+# ==================== Python Build Base ====================
+
+FROM ghcr.io/storytold/docker-base-images-nvidia-cuda-experimental:080f96bc7087 as pybuild-base
+
+# The downloader needs youtube-dl as a python library
+# NB: No longer maintained: https://news.ycombinator.com/item?id=28319624
+#RUN pip3 install --upgrade youtube-dl
+
+# Used by the downloader script.
+# And it looks like youtube-dl is dead...
+# Per the README, it looks like pip will always pull the latest version.
+# https://github.com/yt-dlp/yt-dlp#update
+RUN python3 -m pip install --upgrade yt-dlp
+
+# ==================== Python Build Step 2: Wav2Lip Requirements ====================
+
+FROM pybuild-base as pybuild-requirements
+
+WORKDIR /
+
+COPY models/Wav2Lip ./models/Wav2Lip
+WORKDIR models/Wav2Lip
+
+# NB: We need Python3.6
+RUN python3.6 -m venv python
+#RUN virtualenv -p /usr/bin/python3.6  ... wait
+RUN . python/bin/activate \
+  && pip install --upgrade pip \
+  && pip install -r requirements.txt \
+  && deactivate
+
+WORKDIR /
+
+COPY models/tts ./models/tts
+WORKDIR models/tts
+
+# NB: Not sure if we need Python3.6 for Tacotron2, but Python3.8 gave me trouble.
+# NB: Setuptools fix: https://github.com/tensorflow/tensorflow/issues/34302#issuecomment-554450289
+RUN python3.6 -m venv python
+RUN . python/bin/activate \
+  && pip install --upgrade pip setuptools \
+  && pip install -r requirements-tacotron-python36.txt \
+  && deactivate
+
+# ==================== Rust Build Base ====================
+
+# Custom base image
+# Make sure to add this repository so it has read acces to the base image:
+# https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access
+FROM ghcr.io/storytold/docker-base-images-rust-ssl:latest as rust-build
+WORKDIR /tmp
+
+COPY Cargo.lock . 
+COPY Cargo.toml .
+COPY sqlx-data.json .
+COPY crates/ ./crates
+COPY db/ ./db
+
+RUN $HOME/.cargo/bin/cargo fetch
+
+ARG GIT_SHA
+RUN echo -n ${GIT_SHA} > GIT_SHA
+
+# Run all of the tests
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo test
+
+# Build all the binaries.
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin storyteller-web
+
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin tts-download-job
+
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin w2l-download-job
+
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin generic-download-job
+
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin tts-inference-job
+
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin w2l-inference-job
+
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin websocket-gateway
+
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin twitch-pubsub-subscriber
+
+# Final image
+#  FROM ubuntu:xenial
+FROM pybuild-requirements as final
+
+WORKDIR /
+
+COPY scripts/ ./scripts
+
+# See: https://github.com/opencontainers/image-spec/blob/master/annotations.md
+LABEL org.opencontainers.image.authors='bt@brand.io, echelon@gmail.com'
+LABEL org.opencontainers.image.url='https://github.com/storytold/storyteller-web'
+LABEL org.opencontainers.image.documentation='https://github.com/storytold/storyteller-web'
+LABEL org.opencontainers.image.source='https://github.com/storytold/storyteller-web'
+
+# NB(echelon,2022-05-31): NVIDIA revoked some of their keys.
+# A better long term fix is to remove these dependencies outright and not have ML deps in this repo.
+# https://developer.nvidia.com/blog/updating-the-cuda-linux-gpg-repository-key/
+# https://forums.developer.nvidia.com/t/invalid-public-key-for-cuda-apt-repository/212901/11
+RUN apt-key del 7fa2af80
+
+# NB: Fix for 'E: Conflicting values set for option Signed-By regarding source'
+# NB: We need to remove all sources
+RUN rm $(find /etc/apt | grep cuda)
+
+#RUN wget https://developer.download.nvidia.com/compute/cuda/repos/$distro/$arch/cuda-keyring_1.0-1_all.deb
+RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb
+RUN dpkg -i cuda-keyring_1.0-1_all.deb
+
+# NB: Comment this out for non-debug images
+# TODO: Figure out how this is done elsewhere with just the static binaries
+# Others: `mysql-client`
+RUN apt-get update \
+    && apt-get install -y \
+        curl \
+        python3-pip \
+        python3 \
+        wget
+
+RUN python3 --version
+
+# TODO: Use venv to do this instead.
+RUN pip3 install gdown
+
+# Copy all the binaries.
+COPY --from=rust-build /tmp/target/release/storyteller-web /
+COPY --from=rust-build /tmp/target/release/tts-download-job /
+COPY --from=rust-build /tmp/target/release/tts-inference-job /
+COPY --from=rust-build /tmp/target/release/w2l-download-job /
+COPY --from=rust-build /tmp/target/release/w2l-inference-job /
+COPY --from=rust-build /tmp/target/release/websocket-gateway /
+COPY --from=rust-build /tmp/target/release/twitch-pubsub-subscriber /
+COPY --from=rust-build /tmp/target/release/generic-download-job /
+COPY --from=rust-build /tmp/GIT_SHA /
+
+# SSL certs are required for crypto
+COPY --from=rust-build /etc/ssl /etc/ssl
+
+# Required dynamically linked libraries
+COPY --from=rust-build /lib/x86_64-linux-gnu/libssl.*             /lib/x86_64-linux-gnu/
+COPY --from=rust-build /lib/x86_64-linux-gnu/libcrypto.*          /lib/x86_64-linux-gnu/
+
+# Make sure all the links resolve
+RUN ldd storyteller-web
+
+# Without a .env file, Rust crashes "mysteriously" (ugh)
+RUN touch .env
+RUN touch .env-secrets
+
+EXPOSE 8080
+CMD LD_LIBRARY_PATH=/usr/lib /storyteller-web

README.md

@@ -1,46 +1,77 @@
 storyteller-rust
 ================
 
-Our Rust monorepo, which serves both our API surface area and our downloadable client applications.
+Our Rust monorepo, which services our 
+API surface area. 
+
+Application overview
+--------------------
+
+* [FakeYou.com](https://fakeyou.com) is powered by:
+  * HTTP APIs:
+    * `storyteller-web` - HTTP + database monolith for FakeYou
+  * Asynchronous Jobs:
+    * `tts-download-job` - Async download of models
+    * `tts-inference-job` - Async TTS inference
+    * `w2l-download-job` - Async download of videos and images
+    * `w2l-inference-job` - Async W2L inference
+
+* [StorytellerStream](https://storyteller.stream) is powered by:
+  * HTTP APIs:
+    * `storyteller-web` - HTTP + database monolith for FakeYou
+  * Websocket events:
+    * `websocket-gateway` - Websocket to pull down Twitch events.
+  * Twitch API integration:
+    * `twitch-pubsub-subscriber` - Dynamically subscribes to Twitch PubSub for "active" users
+
+Schema and API notes
+--------------------
+
+* [Redis Schema Notes README](_docs/redis_schema.md)
+
+Local development
+-----------------
+
+[Local development setup README](_docs/local_development_setup.md)
+
+Production and Deployment
+-------------------------
+
+[Production builds and deployment README](_docs/production_builds_and_deployment.md)
+
+Development notes
+-----------------
+
+[Development notes README](_docs/development_notes.md)
+
+Dump a MySQL report
+-------------------
 
 ```
-                 ."-,.__
-                 `.     `.  ,
-              .--'  .._,'"-' `.
-             .    .'         `'
-             `.   /          ,'
-               `  '--.   ,-"'
-                `"`   |  \
-                   -. \, |
-                    `--Y.'      ___.
-                         \     L._, \
-               _.,        `.   <  <\                _
-             ,' '           `, `.   | \            ( `
-          ../, `.            `  |    .\`.           \ \_
-         ,' ,..  .           _.,'    ||\l            )  '".
-        , ,'   \           ,'.-.`-._,'  |           .  _._`.
-      ,' /      \ \        `' ' `--/   | \          / /   ..\
-    .'  /        \ .         |\__ - _ ,'` `        / /     `.`.
-    |  '          ..         `-...-"  |  `-'      / /        . `.
-    | /           |L__           |    |          / /          `. `.
-   , /            .   .          |    |         / /             ` `
-  / /          ,. ,`._ `-_       |    |  _   ,-' /               ` \
- / .           \"`_/. `-_ \_,.  ,'    +-' `-'  _,        ..,-.    \`.
-.  '         .-f    ,'   `    '.       \__.---'     _   .'   '     \ \
-' /          `.'    l     .' /          \..      ,_|/   `.  ,'`     L`
-|'      _.-""` `.    \ _,'  `            \ `.___`.'"`-.  , |   |    | \
-||    ,'      `. `.   '       _,...._        `  |    `/ '  |   '     .|
-||  ,'          `. ;.,.---' ,'       `.   `.. `-'  .-' /_ .'    ;_   ||
-|| '              V      / /           `   | `   ,'   ,' '.    !  `. ||
-||/            _,-------7 '              . |  `-'    l         /    `||
-. |          ,' .-   ,' ||               | .-.        `.      .'     ||
- `'        ,'    `".'    |               |    `.        '. -.'       `'
-          /      ,'      |               |,'    \-.._,.'/'
-          .     /        .               .       \    .''
-        .`.    |         `.             /         :_,'.'
-          \ `...\   _     ,'-.        .'         /_.-'                       S T O R Y T E L L E R
-           `-.__ `,  `'   .  _.>----''.  _  __  /                        Turning memes into dreams
-                .'        /"'          |  "'   '_                      One Rust Monorepo at a time
-               /_|.-'\ ,".             '.'`__'-( \
-                 / ,"'"\,'               `/  `-.|" 
-```
+mysql -u storyteller \
+  -pPasswordHere \
+  -h IP_HERE \
+  -P 3306 \
+  -D storyteller \
+  -B \
+  -e "select * from voice_clone_requests;" | sed "s/'/\'/;s/\t/\",\"/g;s/^/\"/;s/$/\"/;s/\n//g" > voice_clone_report.csv
+```
+
+TODO
+----
+
+* Revise this README to be more useful.
+
+Notes / TODOs:
+
+* Examples for good Actix+Sqlx Tests:
+  https://stackoverflow.com/questions/65370752/how-do-i-create-an-actix-web-server-that-accepts-both-sqlx-database-pools-and-tr
+
+* Actix/sqlx runtime compat:
+  https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734
+  
+* Redis caching of more endpoints
+
+* Jobs for analytics queries (#uses per model, etc)
+* Jobs for ingesting Discord, Twitter, Reddit RPAN chat, Twitch chat (not PubSub), etc.
+

_develop/docs/aichatbot_sidecar.md

@@ -1,38 +0,0 @@
-AI Chatbot Sidecar
-==================
-
-This is the first app that doesn't run as a part of our SoA API architecture. (Though it is written such that we 
-could port it.) This app is responsible for generating all the content that goes into our streaming AI-powered TV
-programs.
-
-### Windows Installation
-
-The application uses sqlite, which is tricky to install on Windows.
-
-1. Put together an sqlite build directory containing the sources and dll file. One is checked in under this monorepo 
-   in `./windows/sqlite` at the time of this writing, though that may change in the future.
-
-
-2. Add environment variables for the sqlite build directory (see https://github.com/diesel-rs/diesel/issues/487)
-
-    * PATH (append the sqlite build directory to the end)
-    * SQLITE3_LIB_DIR (a new environment variable; set it to the sqllite build path)
-
-
-3. Build the `sqlite3.lib` file with the following:
-
-   ```
-   lib /def:sqlite3.def /out:sqlite3.lib /MACHINE:amd64 
-   ```
-   
-   This must be done in "Developer Command Prompt for VS 2022" (or similar), not `cmd` or `PowerShell`.
-
-
-4. Run the diesel_cli tool build  
-   ```
-   cargo install diesel_cli --no-default-features --features sqlite
-   ```
-
-   If it still complains about the `sqlite3.lib` file not being found, be sure to double check that `echo %PATH%` shows 
-   the updated PATH from the steps above.
-

_develop/docs/local_development_k8s.md

@@ -1,20 +0,0 @@
-# Local Kubernetes environment for standing up the APIs and their dependencies
-
-- Install minikube, kubectl, docker, and musl from your OS package manager
-- `rustup target add x86_64-unknown-linux-musl`
-- `cargo build --release --bin storyteller-web --target=x86_64-unknown-linux-musl`
-- `minikube start --extra-config=apiserver.service-node-port-range=1-65535`
-- `eval $(minikube docker-env)`
-- `minikube mount /path/to/storyteller-rust/:/storyteller-rust &> /dev/null &`
-- `docker build -t storyteller-web -f localdev/Dockerfile .`
-- `kubectl apply -f localdev/kubernetes.yml`
-- Get the ip address of minikube with `minikube ip`, this is where all the services will be running on their 
-regular ports (12345 for the api, 6379 for redis, 3306 for mysql)
-- That's it, we just spun up mysql, applied database migrations, spun up redis, and spun up our api
-
-# Development tips
-- Rebuild and "deploy" (locally) with `cargo build --release --bin storyteller-web --target=x86_64-unknown-linux-musl && kubectl rollout restart storyteller-web`
-- View logs from the api with `kubectl logs  storyteller-web<TAB>`
-- Delete everything minikube (if you start running out of disk space or what-have-you) with `minikube delete`
-- See pod status with `kubectl get pods`
-- If there are errors you can get more info with `kubectl describe pods`

_develop/docs/old_readme.md

@@ -1,77 +0,0 @@
-old readme
-==========
-
-Contents of the main README.md, which will always be woefully out of date. Docs will migrate into this folder on a 
-per-subject basis. App-specific docs will live under each app.
-
-Application overview
---------------------
-
-* [FakeYou.com](https://fakeyou.com) is powered by:
-    * HTTP APIs:
-        * `storyteller-web` - HTTP + database monolith for FakeYou
-    * Asynchronous Jobs:
-        * `tts-download-job` - Async download of models
-        * `tts-inference-job` - Async TTS inference
-        * `w2l-download-job` - Async download of videos and images
-        * `w2l-inference-job` - Async W2L inference
-
-* [StorytellerStream](https://storyteller.stream) is powered by:
-    * HTTP APIs:
-        * `storyteller-web` - HTTP + database monolith for FakeYou
-    * Websocket events:
-        * `websocket-gateway` - Websocket to pull down Twitch events.
-    * Twitch API integration:
-        * `twitch-pubsub-subscriber` - Dynamically subscribes to Twitch PubSub for "active" users
-
-Schema and API notes
---------------------
-
-* [Redis Schema Notes README](_docs/redis_schema.md)
-
-Local development
------------------
-
-[Local development setup README](_docs/local_development_setup.md)
-
-Production and Deployment
--------------------------
-
-[Production builds and deployment README](_docs/production_builds_and_deployment.md)
-
-Development notes
------------------
-
-[Development notes README](_docs/development_notes.md)
-
-Dump a MySQL report
--------------------
-
-```
-mysql -u storyteller \
-  -pPasswordHere \
-  -h IP_HERE \
-  -P 3306 \
-  -D storyteller \
-  -B \
-  -e "select * from voice_clone_requests;" | sed "s/'/\'/;s/\t/\",\"/g;s/^/\"/;s/$/\"/;s/\n//g" > voice_clone_report.csv
-```
-
-TODO
-----
-
-* Revise this README to be more useful.
-
-Notes / TODOs:
-
-* Examples for good Actix+Sqlx Tests:
-  https://stackoverflow.com/questions/65370752/how-do-i-create-an-actix-web-server-that-accepts-both-sqlx-database-pools-and-tr
-
-* Actix/sqlx runtime compat:
-  https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734
-
-* Redis caching of more endpoints
-
-* Jobs for analytics queries (#uses per model, etc)
-* Jobs for ingesting Discord, Twitter, Reddit RPAN chat, Twitch chat (not PubSub), etc.
-

_develop/localdev/Dockerfile

@@ -1,23 +0,0 @@
-FROM rust:latest
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && \
-    apt-get install -y \
-        mariadb-client \
-        imagemagick  \
-        libmariadb-dev \
-    && rm -rf /var/lib/apt/lists/*
-
-
-RUN cargo install diesel_cli --features mysql && \
-  cargo install sqlx-cli --features mysql
-
-WORKDIR /storyteller-rust
-
-ARG GIT_SHA
-
-RUN echo -n ${GIT_SHA} > /GIT_SHA
-
-CMD ["/storyteller-rust/localdev/startup.sh"]
-

_develop/localdev/kubernetes.yml

@@ -1,153 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata: 
-  name: mysql
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: mysql
-  template:
-    metadata:
-      labels:
-        app: mysql
-    spec:
-      containers:
-      - name: mysql
-        image: mysql:8.0.30
-        env:
-        - name: MYSQL_USER
-          value: storyteller
-        - name: MYSQL_PASSWORD
-          value: password
-        - name: MYSQL_DATABASE
-          value: storyteller
-        - name: MYSQL_ROOT_PASSWORD
-          value: password
-        ports:
-          - containerPort: 3306 
-
----
-
-apiVersion: apps/v1
-kind: Deployment
-metadata: 
-  name: redis
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: redis
-  template:
-    metadata:
-      labels:
-        app: redis
-    spec:
-      containers:
-      - name: redis
-        image: redis:7.0.4
-        ports:
-          - containerPort: 6379
-
----
-
-apiVersion: apps/v1
-kind: Deployment
-metadata: 
-  name: storyteller-web
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: storyteller-web
-  template:
-    metadata:
-      labels:
-        app: storyteller-web
-    spec:
-      containers:
-      - name: storyteller-web
-        image: storyteller-web:latest
-        imagePullPolicy: Never
-        env:
-          - name: MYSQL_URL
-            value: mysql://storyteller:password@mysql.default/storyteller
-          - name: REDIS_0_URL
-            value: redis://redisd@redis.default/0
-          - name: REDIS_1_URL
-            value: redis://redisd@redis.default/1
-          - name: ACCESS_KEY
-            value: garbage
-          - name: SECRET_KEY
-            value: garbage
-          - name: REGION_NAME
-            value: garbage
-          - name: W2L_PRIVATE_DOWNLOAD_BUCKET_NAME
-            value: garbage
-          - name: W2L_PUBLIC_DOWNLOAD_BUCKET_NAME
-            value: garbage
-          - name: AUDIO_UPLOADS_BUCKET_ROOT
-            value: garbage
-          - name: TWITCH_USERNAME
-            value: garbage
-          - name: TWITCH_STREAM_KEY
-            value: garbage
-          - name: TWITCH_APP_CLIENT_ID
-            value: garbage
-          - name: TWITCH_APP_CLIENT_SECRET
-            value: garbage
-        ports:
-          - containerPort: 12345
-        volumeMounts:
-          - mountPath: /storyteller-rust
-            name: storyteller-rust-folder
-      volumes:
-        - name: storyteller-rust-folder
-          hostPath:
-            path: /storyteller-rust
----
-
-apiVersion: v1
-kind: Service
-metadata: 
-  name: mysql
-spec:
-  type: NodePort
-  ports:
-    - name: mysql
-      port: 3306
-      nodePort: 3306
-  selector:
-    app: mysql
-
----
-
-apiVersion: v1
-kind: Service
-metadata: 
-  name: redis
-spec:
-  type: NodePort
-  ports:
-    - name: redis
-      port: 6379
-      nodePort: 6379
-  selector:
-    app: redis
-
----
-
-apiVersion: v1
-kind: Service
-metadata: 
-  name: storyteller-web
-spec:
-  type: NodePort
-  ports:
-    - name: storyteller-web
-      port: 12345
-      nodePort: 12345
-  selector:
-    app: storyteller-web
-
-

_develop/localdev/nginx-http-config/storyteller-ssl

@@ -1,258 +0,0 @@
-# Nginx configs for "jungle.horse" and "api.jungle.horse" (http)
-#
-# On Ubuntu Linux, this config is located at the following location:
-# 
-#  - /etc/nginx/sites-enabled/{name-of-site}
-#
-# Move this file there, and then restart with,
-#
-#   sudo service nginx restart
-#
-# On Mac, this is located at the following locations:
-#
-#  - /opt/homebrew/etc/nginx/servers/{name-of-site} (for nginx installed with homebrew)
-#  - /usr/local/etc/nginx/sites-enabled/{name-of-site} (for default nginx install)
-#
-#
-# On Mac, verify configs and restart with,
-#
-#   sudo nginx -t
-#   sudo nginx -s reload
-#
-#           ================
-#           SSL Instructions
-#           ================
-#
-# You can launch Chrome in a way it disregards self-signed certs and CORS:
-#
-#     chromium  --disable-web-security --ignore-certificate-errors --user-data-dir=~/chromeTemp
-#
-# To create self-signed certs on Ubuntu Linux,
-#
-#     sudo apt-get install openssl
-#
-#     sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/localhost.key -out /etc/ssl/certs/localhost.crt
-#
-#     ^ most fields can be blank, but the "Common Name" should be `*.jungle.horse`
-#
-#     sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/devfakeyou.key -out /etc/ssl/certs/devfakeyou.crt
-#     sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/devstoryteller.key -out /etc/ssl/certs/devstoryteller.crt
-#
-#     ^ most fields can be blank, but the "Common Name" should be `*.dev.fakeyou.com` and `*.dev.storyteller.io` respectively.
-#
-# For more: https://www.cloudsavvyit.com/1306/how-to-create-and-use-self-signed-ssl-on-nginx/
-#
-# To create self-signed certs on Mac,
-#
-#     openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt
-#
-#     ^ most fields can be blank, but the "Common Name" should be `*.jungle.horse`
-#
-# On Mac, load the self signed cert into the keychain with the following command:
-#
-#     sudo security add-trusted-cert \
-#      -d -r trustRoot \
-#      -k /Library/Keychains/System.keychain /etc/ssl/certs/localhost.crt
-#
-# Self signed certs might be considered insecure by the browser. In Firefox, 
-# set about:config `security.tls.insecure_fallback_hosts` to include a list of
-# permitted domains. You may also need to configure other settings. 
-#
-#
-
-# ========================================================
-#                      FAKEYOU
-# ========================================================
-
-# FakeYou javascript frontend over plain HTTP (launched by Yarn)
-server {
-    listen 80;
-    server_name dev.fakeyou.com;
-    location / {
-        proxy_set_header Host $host;
-        proxy_pass http://127.0.0.1:7000;
-        proxy_redirect off;
-    }
-}
-
-# FakeYou API backend over plain HTTP (Rust app)
-server {
-    listen 80;
-    server_name api.dev.fakeyou.com;
-    location / {
-        proxy_set_header Host $host;
-        proxy_pass http://127.0.0.1:12345;
-        proxy_redirect off;
-    }
-}
-
-# FakeYou javascript frontend over SSL (launched by Yarn)
-server {
-    listen 443 ssl;
-    listen [::]:443 ssl;
-    ssl_certificate /etc/ssl/certs/devfakeyou.crt;
-    ssl_certificate_key /etc/ssl/private/devfakeyou.key;
-    ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
-    server_name dev.fakeyou.com;
-    location / {
-        proxy_set_header Host $host;
-        proxy_pass http://127.0.0.1:7000;
-        proxy_redirect off;
-    }
-}
-
-# FakeYou API backend over SSL (Rust app)
-server {
-    listen 443 ssl;
-    listen [::]:443 ssl;
-    ssl_certificate /etc/ssl/certs/devfakeyou.crt;
-    ssl_certificate_key /etc/ssl/private/devfakeyou.key;
-    ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
-    server_name api.dev.fakeyou.com;
-    location / {
-        proxy_set_header Host $host;
-        proxy_pass http://127.0.0.1:12345;
-        proxy_redirect off;
-    }
-}
-
-# ========================================================
-#                      STORYTELLER
-# ========================================================
-
-# Storyteller javascript frontend over plain HTTP (launched by Yarn)
-server {
-    listen 80;
-    server_name dev.storyteller.io;
-    location / {
-        proxy_set_header Host $host;
-        proxy_pass http://127.0.0.1:7000;
-        proxy_redirect off;
-    }
-}
-
-# Storyteller API backend over plain HTTP
-server {
-    listen 80;
-    server_name api.dev.storyteller.io;
-    location / {
-        proxy_set_header Host $host;
-        proxy_pass http://127.0.0.1:12345;
-        proxy_redirect off;
-    }
-}
-
-# Storyteller javascript frontend over SSL (launched by Yarn)
-server {
-    listen 443 ssl;
-    listen [::]:443 ssl;
-    ssl_certificate /etc/ssl/certs/devstoryteller.crt;
-    ssl_certificate_key /etc/ssl/private/devstoryteller.key;
-    ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
-    server_name dev.storyteller.io;
-    location / {
-        proxy_set_header Host $host;
-        proxy_pass http://127.0.0.1:7000;
-        proxy_redirect off;
-    }
-}
-
-# Storyteller API backend over SSL
-server {
-    listen 443 ssl;
-    listen [::]:443 ssl;
-    ssl_certificate /etc/ssl/certs/devstoryteller.crt;
-    ssl_certificate_key /etc/ssl/private/devstoryteller.key;
-    ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
-    server_name api.dev.storyteller.io;
-    location / {
-        proxy_set_header Host $host;
-        proxy_pass http://127.0.0.1:12345;
-        proxy_redirect off;
-    }
-}
-
-# ========================================================
-#            OLD PRODUCTS THAT NEED UPDATING
-# ========================================================
-
-#     ## TODO(2022-02-13): This is new and untested
-#     # WebSocket API backend over plain HTTP
-#     server {
-#         listen 80;
-#         server_name ws.jungle.horse;
-#         location / {
-#             proxy_set_header Host $host;
-#             proxy_pass http://127.0.0.1:54321;
-#             proxy_redirect off;
-#         }
-#     }
-#
-#     # Websocket API backend over SSL
-#     server {
-#         listen 443 ssl;
-#         listen [::]:443 ssl;
-#         ssl_certificate /etc/ssl/certs/localhost.crt;
-#         ssl_certificate_key /etc/ssl/private/localhost.key;
-#         ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
-#         server_name ws.jungle.horse;
-#         location / {
-#             proxy_set_header Host $host;
-#             proxy_pass http://127.0.0.1:54321;
-#             proxy_redirect off;
-#         }
-#     }
-#
-#     # Javascript frontend over plain HTTP
-#     server {
-#         listen 80;
-#         server_name jungle.horse;
-#         location / {
-#             proxy_set_header Host $host;
-#             proxy_pass http://127.0.0.1:7000;
-#             proxy_redirect off;
-#         }
-#     }
-#
-#     # TODO(2022-02-13): This is new and untested
-#     # Javascript frontend (OBS Layer) over plain HTTP
-#     server {
-#         listen 80;
-#         server_name obs.jungle.horse;
-#         location / {
-#             proxy_set_header Host $host;
-#             proxy_pass http://127.0.0.1:7007;
-#             proxy_redirect off;
-#         }
-#     }
-#
-#     # Javascript frontend over SSL
-#     server {
-#         listen 443 ssl;
-#         listen [::]:443 ssl;
-#         ssl_certificate /etc/ssl/certs/localhost.crt;
-#         ssl_certificate_key /etc/ssl/private/localhost.key;
-#         ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
-#         server_name jungle.horse;
-#         location / {
-#             proxy_set_header Host $host;
-#             proxy_pass http://127.0.0.1:7000;
-#             proxy_redirect off;
-#         }
-#     }
-#
-#     # Javascript frontend over SSL (OBS Layer)
-#     server {
-#         listen 443 ssl;
-#         listen [::]:443 ssl;
-#         ssl_certificate /etc/ssl/certs/localhost.crt;
-#         ssl_certificate_key /etc/ssl/private/localhost.key;
-#         ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
-#         server_name obs.jungle.horse;
-#         location / {
-#             proxy_set_header Host $host;
-#             proxy_pass http://127.0.0.1:7007;
-#             proxy_redirect off;
-#         }
-#     }
-#

_develop/localdev/startup.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-set +e +o pipefail
-diesel migration run --database-url="${MYSQL_URL}"
-set -e -o pipefail
-./target/x86_64-unknown-linux-musl/release/storyteller-web
-

_develop/localdev/ubuntu_setup.sh

@@ -1,36 +0,0 @@
-#!/bin/bash
-
-echo 'Installing APT packages...'
-
-# OpenSSL requirements: libssl-dev, pkg-config
-# Database requirements: mysql-server, libmysqlclient-dev
-# Other tools: jq (db codegen)
-sudo apt-get install -y \
-  jq \
-  libmysqlclient-dev \
-  libssl-dev \
-  mysql-server \
-  pkg-config \
-  --no-install-recommends
-
-echo 'Installing Cargo packages...'
-
-# Install Diesel CLI for migrations
-cargo install diesel_cli \
-  --no-default-features \
-  --features mysql
-
-# Install SQLx for codegen
-cargo install sqlx-cli \
-  --no-default-features \
-  --features rustls,mysql
-
-echo 'Installing Database user...'
-
-# Create DB account (using default system mysql password)
-sudo mysql \
-  -u "root" \
-  -p"password" \
-  -e "use mysql; CREATE DATABASE IF NOT EXISTS storyteller; CREATE USER IF NOT EXISTS 'storyteller'@'localhost' IDENTIFIED BY 'password'; GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';"
-
-

_develop/tools/.gitignore

@@ -1,6 +0,0 @@
-# Secrets
-db_secrets.toml
-
-# Python Virtual Env
-python/
-

_develop/tools/find_101soundboards_accounts.py

@@ -1,100 +0,0 @@
-#!/usr/bin/env python
-
-# FIXME: Needs packages 'toml' and 'mysql-connector-python'. Put in requirements.txt.
-
-import time
-import toml
-import mysql.connector
-
-DB_SECRETS_FILE = 'db_secrets.toml'
-
-# TODO: Share db-related setup when more scripts are added.
-
-def read_database_secrets(secrets_file = DB_SECRETS_FILE):
-    secrets = None
-    with open(secrets_file, "r") as f:
-        secrets = toml.load(f)
-
-    required_keys = ['database', 'hostname', 'port', 'password', 'username']
-    for key in required_keys:
-        if key not in secrets:
-            raise Exception(f"Required key '#{key}' not present")
-
-    return secrets
-
-def connect_to_mysql():
-    secrets = read_database_secrets()
-    connection = mysql.connector.connect(
-        host = secrets['hostname'],
-        port = secrets['port'],
-        user = secrets['username'],
-        password = secrets['password'],
-        database = secrets['database'],
-    )
-    return connection
-
-connection = connect_to_mysql()
-cursor = connection.cursor()
-
-SAFE_ID = 76000000 # Reasonably completed on 2022-06-24
-LIMIT = 100000
-
-#def get_record_count():
-#    query = f"SELECT COUNT(*) FROM tts_inference_jobs"
-#    cursor.execute(query)
-#    result = cursor.fetchall()
-#    print(result)
-#    return result[0]
-#
-##total_count = get_record_count()
-
-#total_count = 75082057
-total_count = 21925602
-
-def get_user_tokens(cursor):
-    query = f"""
-		SELECT
-			r.maybe_creator_user_token, r.creator_ip_address
-		FROM
-			tts_results as r
-				LEFT OUTER JOIN
-			users as u
-			ON
-					r.maybe_creator_user_token = u.token
-		WHERE
-				r.raw_inference_text LIKE '%spidermant%'
-		AND r.created_at > ( CURDATE() - INTERVAL 1 HOUR )
-    """
-    cursor.execute(query)
-    results = cursor.fetchall()
-
-    user_tokens = [row[0] for row in results]
-    user_tokens = [token for token in user_tokens if token is not None]
-    user_tokens = set(user_tokens)
-
-    ip_addresses = [row[1] for row in results]
-    ip_addresses = [ip for ip in ip_addresses if ip is not None]
-    ip_addresses = set(ip_addresses)
-
-    return (user_tokens, ip_addresses)
-
-(user_tokens, ip_addresses) = get_user_tokens(cursor)
-
-print(user_tokens)
-print(len(user_tokens))
-print(len(ip_addresses))
-
-with open('101_user_tokens.txt', 'w') as f:
-    for user_token in user_tokens:
-        f.write(user_token)
-        f.write("\n")
-
-with open('101_ip_addresses.txt', 'w') as f:
-    for ip in ip_addresses:
-        f.write(ip)
-        f.write("\n")
-
-
-
-
-

_develop/tools/requirements.txt

@@ -1,3 +0,0 @@
-mysql-connector-python==8.0.32
-protobuf==3.20.3
-toml==0.10.2

_docs/local_development_setup.md

@@ -1,9 +1,9 @@
 dev setup
 =========
 
-[Back to main README](../../README.md)
+[Back to main README](../README.md)
 
-## Debugging and Fixing CUDA/PyTorch
+### Debugging and Fixing CUDA/PyTorch
 
 See this spreadsheet for previous battles with version incompatibility:
 https://docs.google.com/spreadsheets/d/1BEdLmwOzo3r83-iJn9sj6co1VT92t3lIeJ-lNH25bdQ/edit#gid=0
@@ -11,53 +11,27 @@ https://docs.google.com/spreadsheets/d/1BEdLmwOzo3r83-iJn9sj6co1VT92t3lIeJ-lNH25
 Ubuntu might upgrade the driver by accident, and it might need reinstallation. Who knows.
 This stuff is a nightmare.
 
-## Database Setup
-
-### Mac Install
-
-```
-brew install mysql
-# If mysql -uroot fails, reboot the machine:
-sudo reboot now
-```
-
-### Linux Install
+### Database Setup
 
 Install the following libraries, and see the notes further below about MySQL on Ubuntu 20.04.
 
 ```
-jq # for combining mysql codegen outputs
-libmysqlclient-dev
 mysql-server
-libsqlite3-dev
-pkgconf # if using openssl instead of rustls
+imagemagick
 ```
 
-### Migrations
-
 To manage the database and perform migrations, install the Rust tools diesel and sqlx.
 
 We'll be using diesel to manage the migrations, but sqlx within the app to actually perform queries.
 Diesel is an ORM, which is dumb, so we use sqlx as at-compile-time typesafe SQL.
 
-The CLI is necessary to perform codegen:
-
 ```
 sudo apt-get install libmysqlclient-dev
-cargo install sqlx-cli --no-default-features --features rustls,mysql,sqlite
+cargo install sqlx-cli --no-default-features --features mysql [2022-01-16: is this needed?]
 ```
 
-Diesel now supports configuring the migration directory via an environment variable,
-so we can install the currently published version:
-
-```
-cargo install diesel_cli \
-  --no-default-features \
-  --features mysql,sqlite
-```
-
-If the modern version fails, try installing a cherry-picked version with a 
-TOML-based migration directory configuration:
+To install a more modern version of Diesel (unreleased) that 
+supports the "migrations_dir" toml config option:
 
 ```
 cargo install diesel_cli \
@@ -86,21 +60,6 @@ If MySql in local dev can't be connected to, reset the accounts:
 
 https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux
 
-#### MySql on Ubuntu 22.04
-
-Should work largely out of the box. We'll need a dev account:
-
-```
-# Connect to mysql:
-# sudo mysql -u root -p (password is "root")
-
-use mysql;
-CREATE USER 'storyteller'@'localhost' IDENTIFIED BY 'password';
-GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';
-```
-
-Then verify with `./dev_mysql_connect.sh`
-
 #### Fixing dev MySql on Ubuntu 20.04
 
 For some reason, the MySql default install on 20.04 gave me a bunch of trouble.

_migrations/2021-05-15-071534_users/up.sql

@@ -0,0 +1,235 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+CREATE TABLE users (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Visible "primary key"
+  token VARCHAR(32) NOT NULL,
+
+  -- Username is a lookup key; display_name allows the user to add custom case.
+  username VARCHAR(20) NOT NULL,
+  display_name VARCHAR(20) NOT NULL,
+
+  email_address VARCHAR(255) NOT NULL,
+  email_confirmed BOOLEAN NOT NULL DEFAULT false,
+
+  -- The role assigned to the user confers permissions.
+  user_role_slug VARCHAR(16) NOT NULL,
+
+  -- ========== CREDENTIALS ==========
+
+  -- Bcrypt password hash. Granted, there are newer methods:
+  -- https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html
+  -- Passwords may be a max of 64 characters per BCrypt.
+  password_hash BINARY(60) NOT NULL,
+
+  -- Incremented with every update to the password.
+  password_version INT NOT NULL DEFAULT 0,
+
+  -- ========== ABUSE TRACKING ==========
+
+  -- For abuse tracking.
+  -- Wide enough for IPv4/6
+  ip_address_creation VARCHAR(40) NOT NULL,
+  ip_address_last_login VARCHAR(40) NOT NULL,
+  ip_address_last_update VARCHAR(40) NOT NULL,
+
+  -- ========== DISPLAY / PROFILE ==========
+
+  -- Gravatar image hashes are precomputed.
+  email_gravatar_hash CHAR(32) NOT NULL,
+
+  -- The profile of the user in markdown (user editable).
+  profile_markdown TEXT NOT NULL,
+
+  -- Generated HTML (not user-editable).
+  profile_rendered_html TEXT NOT NULL,
+
+  -- An uploaded avatar. Public hash in our bucket.
+  -- avatar_public_bucket_hash CHAR(32) DEFAULT NULL,
+
+  -- ========== SOCIAL MEDIA ==========
+
+  -- Social media usernames
+  -- These are not confirmed. We'll need to build an OAuth system to handle that.
+  discord_username VARCHAR(36) DEFAULT NULL,
+  twitter_username VARCHAR(36) DEFAULT NULL,
+  twitch_username VARCHAR(36) DEFAULT NULL,
+  patreon_username VARCHAR(36) DEFAULT NULL,
+  github_username VARCHAR(36) DEFAULT NULL,
+  cashapp_username VARCHAR(36) DEFAULT NULL,
+  website_url VARCHAR(255) DEFAULT NULL,
+
+  -- ========== USER PREFERENCES ==========
+
+  -- If the user doesn't want to use gravatar and doesn't have an uploaded avatar.
+  disable_gravatar BOOLEAN NOT NULL DEFAULT false,
+
+  -- (THIS MIGHT NOT BE USED)
+  -- NB: DO NOT SORT!
+  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
+  -- Visibility preference used at TTS inference time (which can be changed on the result itself).
+  -- Moderators will still see them.
+  preferred_tts_result_visibility ENUM(
+    'public',
+    'hidden',
+    'private'
+  ) NOT NULL DEFAULT 'public',
+
+  -- (THIS MIGHT NOT BE USED)
+  -- NB: DO NOT SORT!
+  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
+  -- Visibility preference used at TTS inference time (which can be changed on the result itself).
+  -- Moderators will still see them.
+  preferred_w2l_result_visibility ENUM(
+    'public',
+    'hidden',
+    'private'
+  ) NOT NULL DEFAULT 'public',
+
+  -- Auto play preferences
+  -- Tri-state because we want to potentially change the default behavior.
+  auto_play_audio_preference BOOLEAN DEFAULT NULL,
+  auto_play_video_preference BOOLEAN DEFAULT NULL,
+
+  -- Favorite TTS voice to use by default
+  -- maybe_preferred_tts_model_token VARCHAR(32) DEFAULT NULL,
+
+  -- Favorite W2L model to use by default
+  -- maybe_preferred_w2l_template_token VARCHAR(32) DEFAULT NULL,
+
+  -- Settings
+  -- DO NOT REORDER.
+  -- dark_mode_preference ENUM(
+  --   'light-mode',
+  --   'dark-mode',
+  --   'use-clock'
+  --  ) NOT NULL DEFAULT 'light-mode',
+
+  -- ========== STATS ==========
+
+  -- For tracking stats.
+  -- The "cached" values are updated by a background job.
+  -- cached_tts_rendered_counter INT(10) NOT NULL DEFAULT 0,
+  -- cached_w2l_rendered_counter INT(10) NOT NULL DEFAULT 0,
+
+  -- ========== MODERATION DETAILS ==========
+
+  -- Different than deleted.
+  -- Users still show up, but can't do anything.
+  is_banned BOOLEAN NOT NULL DEFAULT false,
+
+  -- If a moderator has comments.
+  maybe_mod_comments VARCHAR(255) DEFAULT NULL,
+
+  -- The last moderator that made changes.
+  maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
+
+  -- ========== VECTOR CLOCK ==========
+
+  -- Incremented with every update.
+  version INT NOT NULL DEFAULT 0,
+
+  -- ========== TIMESTAMPS ==========
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- If the user is deleted, we set this.
+  -- This is different than banned. These users won't show up at all.
+  user_deleted_at TIMESTAMP NULL,
+  mod_deleted_at TIMESTAMP NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (token),
+  UNIQUE KEY (username),
+  UNIQUE KEY (email_address),
+  KEY fk_user_role_slug (user_role_slug)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
+
+CREATE TABLE user_roles (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Effective "primary key"
+  slug CHAR(16) NOT NULL,
+
+  name VARCHAR(255) NOT NULL,
+
+  -- Usage
+  can_use_tts BOOLEAN NOT NULL DEFAULT FALSE,
+  can_use_w2l BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_own_tts_results BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_own_w2l_results BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_own_account BOOLEAN NOT NULL DEFAULT FALSE,
+
+  -- Contribution
+  can_upload_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
+  can_upload_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_own_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_own_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
+
+  -- Moderation
+  can_approve_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
+  -- Edit includes "lock_from_use" and "lock_from_modification"
+  can_edit_other_users_profiles BOOLEAN NOT NULL DEFAULT FALSE,
+  can_edit_other_users_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
+  can_edit_other_users_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_other_users_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_other_users_tts_results BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_other_users_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_other_users_w2l_results BOOLEAN NOT NULL DEFAULT FALSE,
+  can_ban_users BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_users BOOLEAN NOT NULL DEFAULT FALSE,
+
+  -- Incremented with every update.
+  version INT NOT NULL DEFAULT 0,
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (slug)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
+
+CREATE TABLE user_sessions (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Session entropy
+  token VARCHAR(32) NOT NULL,
+
+  -- Foreign key to user
+  user_token VARCHAR(32) NOT NULL,
+
+  -- Track each session's creation IP.
+  -- Wide enough for IPv4/6
+  ip_address_creation VARCHAR(40) NOT NULL,
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- Session termination time.
+  -- This must be set by the server code, or the session is invalid
+  expires_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+  -- deletion = session termination
+  -- Typically these are destroyed by users, but if in the future we allow mods to
+  -- delete them, it doesn't really matter who did the deletion: sessions are not
+  -- designed to be recoverable.
+  deleted_at TIMESTAMP NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (token),
+  KEY fk_user_token (user_token),
+  KEY index_ip_address_creation (ip_address_creation)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_migrations/2022-08-05-202017_generic_download_jobs/up.sql

@@ -0,0 +1,110 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+CREATE TABLE generic_download_jobs (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Effective "primary key" (PUBLIC)
+  -- This is so the in-progress results can be looked up by the UI.
+  token VARCHAR(32) NOT NULL,
+
+  -- Idempotency token
+  -- This is so the frontend client doesn't submit duplicate jobs.
+  uuid_idempotency_token VARCHAR(36) NOT NULL,
+
+  -- ========== SUCCESS CASE ==========
+
+  -- If the job completes successfully, this is the result token.
+  -- This is only populated on a successful result.
+  on_success_downloaded_entity_token VARCHAR(32) DEFAULT NULL,
+
+  -- The type of the object will vary based on the type of the upload,
+  -- and we may include heuristics that auto-detect types in the future
+  on_success_downloaded_entity_type VARCHAR(32) DEFAULT NULL,
+
+  -- ========== UPLOAD DETAILS ==========
+
+  -- A "well defined" type of upload
+  -- The first upload type will be "hifigan_model"
+  download_type VARCHAR(32) NOT NULL,
+
+  -- Assume the same URL can be uploaded more than once (in case of errors or
+  -- people sharing models)
+
+  download_url VARCHAR(512) NOT NULL,
+
+  -- The user-supplied title of the upload
+  -- This might have different meanings for different upload types
+  -- The "name" of the voice model, which might be complicated.
+  -- If maybe_subject_token (etc.) is set, then it's authoritative instead.
+  title VARCHAR(255) NOT NULL,
+
+  -- ========== CREATOR DETAILS AND PREFERENCES ==========
+
+  -- Foreign key to user
+  creator_user_token VARCHAR(32) NOT NULL,
+
+  -- For abuse tracking.
+  -- Wide enough for IPv4/6
+  creator_ip_address VARCHAR(40) NOT NULL,
+
+  -- The creator can set a desired visibility for their data.
+  -- This does not always apply to every upload type.
+  -- Additionally, some upload types may require moderator approval prior
+  -- to being publicly listed, and this field has no bearing on that.
+  -- NB: DO NOT CHANGE THE ORDER OF THE ENUM VALUES DURING SCHEMA MIGRATIONS.
+  creator_set_visibility ENUM(
+    'public',
+    'hidden',
+    'private'
+  ) NOT NULL DEFAULT 'public',
+
+  -- ========== JOB SYSTEM DETAILS ==========
+
+  -- Jobs begin as "pending", then transition to other states.
+  --
+  --  * Pending = job is ready to go
+  --  * Started = job is running
+  --  * Complete_Success = job is done (success)
+  --  * Complete_Failure = job is done (failure)
+  --  * Attempt_Failed = job failed but may retry.
+  --  * Dead = job failed permanently.
+  --
+  -- Pending -> Started -> Complete_Success
+  --                    |-> Complete_Failure
+  --                    \-> Attempt_Failed -> Started -> { Complete, Failed, Dead }
+  status ENUM(
+    'pending',
+    'started',
+    'complete_success',
+    'complete_failure',
+    'attempt_failed',
+    'dead') NOT NULL DEFAULT 'pending',
+
+  -- We can track this against a "max_attempt_count"
+  attempt_count INT(3) NOT NULL DEFAULT 0,
+
+  -- If there is a failure, tell the user why.
+  failure_reason VARCHAR(512) DEFAULT NULL,
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- Failed jobs will set a next attempt time.
+  -- Subsequent tries can increase the timeout.
+  -- Failures because of permissions require human intervention => [retry_at=null].
+  -- Failures because of invalid files are dead => [status=dead].
+  retry_at TIMESTAMP NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (token),
+  UNIQUE KEY (uuid_idempotency_token),
+  KEY fk_on_download_type (download_type),
+  KEY fk_creator_user_token (creator_user_token),
+  KEY index_status (status),
+  KEY index_creator_ip_address (creator_ip_address)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_migrations/2022-08-05-202110_vocoder_models/up.sql

@@ -0,0 +1,128 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+CREATE TABLE vocoder_models (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Effective "primary key" (PUBLIC)
+  token VARCHAR(32) NOT NULL,
+
+  -- The architecture of the vocoder.
+  -- Currently supported values: 'hifigan'
+  vocoder_type VARCHAR(32) NOT NULL,
+
+  -- ========== MUTABLE METADATA ==========
+
+  -- The "name" of the vocoder, which might be complicated.
+  title VARCHAR(255) NOT NULL,
+
+  -- The description of the model in markdown.
+  description_markdown TEXT NOT NULL,
+
+  -- Generated HTML (not user-editable).
+  description_rendered_html TEXT NOT NULL,
+
+  -- ========== MUTABLE MOD-ONLY METADATA ==========
+
+  -- Whether the vocoder is recommended for use
+  is_staff_recommended BOOLEAN NOT NULL DEFAULT FALSE,
+
+  -- ========== IMMUTABLE PROVENANCE AND METADATA ==========
+
+  -- Where the file was originally downloaded (if it was downloaded)
+  original_download_url VARCHAR(512) DEFAULT NULL,
+
+  -- The filename that was used at upload time (if available)
+  original_filename VARCHAR(255) DEFAULT NULL,
+
+  -- File characteristics
+  file_size_bytes INT(10) NOT NULL DEFAULT 0,
+
+  -- ========== CREATOR DETAILS ==========
+
+  -- The person that created the template.
+  creator_user_token VARCHAR(32) NOT NULL,
+
+  -- For abuse tracking.
+  -- Wide enough for IPv4/6
+  creator_ip_address_creation VARCHAR(40) NOT NULL,
+  creator_ip_address_last_update VARCHAR(40) NOT NULL,
+
+  -- (THIS MIGHT NOT BE USED)
+  -- NB: DO NOT SORT!
+  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
+  creator_set_visibility ENUM(
+    'public',
+    'hidden',
+    'private'
+  ) NOT NULL DEFAULT 'public',
+
+  -- ========== BUCKET STORAGE ==========
+
+  -- The pytorch model
+  -- For now, this will be a hash of the file contents.
+  -- NB: NOT UNIQUE! We can allow duplicate uploads.
+  private_bucket_hash CHAR(64) NOT NULL,
+
+  -- The "full url" of the bucket path, not including the hostname
+  private_bucket_object_name VARCHAR(255) NOT NULL,
+
+  -- ========== MODERATION DETAILS ==========
+
+  -- Mods have locked the ability for non-authors (ie. people who didn't upload this)
+  -- to assign the vocoder for use with their TTS models.
+  -- TTS models that already incorporate the vocoder will not be affected.
+  -- The vocoder will remain visible.
+  -- This is independent of other visibility controls (is_mod_disabled_from_author_use,
+  -- deleted_at, and creator_set_visibility)
+  is_mod_disabled_from_public_use BOOLEAN NOT NULL DEFAULT FALSE,
+
+  -- Mods have locked the ability for the author (ie. the person that uploaded this)
+  -- to assign the vocoder for use with their TTS models.
+  -- TTS models that already incorporate the vocoder will not be affected.
+  -- The vocoder will remain visible.
+  -- This is independent of other visibility controls (is_mod_disabled_from_public_use,
+  -- deleted_at, and creator_set_visibility)
+  is_mod_disabled_from_author_use BOOLEAN NOT NULL DEFAULT FALSE,
+
+  -- Extremely popular models may be locked from deletion or modification by users.
+  -- If true, only moderators can edit or delete the vocoder.
+  is_mod_author_editing_locked BOOLEAN NOT NULL DEFAULT FALSE,
+
+  -- If a moderator has comments.
+  maybe_mod_comments VARCHAR(255) DEFAULT NULL,
+
+  -- The last moderator that made changes.
+  maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
+
+  -- ========== VECTOR CLOCK ==========
+
+  -- Incremented with every update.
+  version INT NOT NULL DEFAULT 0,
+
+  -- ========== TIMESTAMPS ==========
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- If this is removed by a mod.
+  -- It completely disappears from the system.
+  user_deleted_at TIMESTAMP NULL,
+  mod_deleted_at TIMESTAMP NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (token),
+  KEY fk_creator_user_token (creator_user_token),
+  KEY fk_maybe_mod_user_token (maybe_mod_user_token),
+  KEY index_creator_ip_address_creation (creator_ip_address_creation),
+  KEY index_creator_ip_address_last_update (creator_ip_address_last_update),
+  KEY index_creator_set_visibility (creator_set_visibility),
+  KEY index_private_bucket_hash (private_bucket_hash),
+  KEY index_is_mod_disabled_from_public_use (is_mod_disabled_from_public_use),
+  KEY index_is_mod_disabled_from_author_use (is_mod_disabled_from_author_use),
+  KEY index_is_mod_author_editing_locked (is_mod_author_editing_locked)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;