windows fixes and performance fixes

unfortunately cpal doesn't support wasm/webaudio input devices https://github.com/RustAudio/cpal/issues/412

.dockerignore

@@ -1,11 +1,9 @@
 *secret*sh
 *~
-.dockerignore
 .env
 .env-secrets
 .git
 .idea/
-Dockerfile
 __pycache__/
 _ignored_schema.rs
 models/Wav2Lip/python/

.env

@@ -1,27 +1,16 @@
-SERVER_ENVIRONMENT=development
-
 DATABASE_URL=mysql://storyteller:password@localhost/storyteller
 #DATABASE_URL=mysql://root:root@localhost/storyteller
 #DATABASE_URL=mysql://root@localhost/storyteller
 
-# Diesel-cli reads this for the migration directory to use.
-# We've renamed it so the migrations are sorted near the top of the project directory tree
-# (since they kind of serve as documentation)
-MIGRATION_DIRECTORY=_sql/migrations/
-
 # SQLX normally requires a live database to reflect upon at build time.
 # This setting turns that on and requires a `sql-data.json` cache file instead.
 # See https://github.com/launchbadge/sqlx/tree/master/sqlx-cli#readme
 #SQLX_OFFLINE=true
 
-# NB(bt, 2023-03-20):
-#  These are used in storyteller-web, image-generation-service, and websocket-gateway.
-#  I'm moving them to env configs that live under storyteller-web/.env.
-#  This will break the other services for now.
-# # We can't serve HTTPS locally without self-signed certificate nonsense.
-# COOKIE_SECURE=false
-# COOKIE_DOMAIN=.jungle.horse
-# WEBSITE_HOMEPAGE_REDIRECT=http://jungle.horse
+# We can't serve HTTPS locally without self-signed certificate nonsense.
+COOKIE_SECURE=false
+COOKIE_DOMAIN=.jungle.horse
+WEBSITE_HOMEPAGE_REDIRECT=http://jungle.horse
 
 # Bucket config (non-secret)
 REGION_NAME=us-east1

.gitattributes

@@ -1,5 +0,0 @@
-# Ignore certain code for Github language statistics
-# https://github.com/github/linguist/blob/master/docs/overrides.md
-old_code/trumpet-trumped.com/dictionary/cmudict-0.7b linguist-vendored
-old_code/voder-vocodes-1.0/frontends/admin/node_modules linguist-vendored
-windows/sqlite linguist-vendored

.github/workflows/docker-publish-cpu.yml

@@ -1,115 +0,0 @@
-# Adapted from
-# https://github.com/actions/starter-workflows/ci/docker-publish.yml
-# If using base images, don't forget to add READ access, eg. from a package settings page like:
-# https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access
-name: "[CPU] Build GHCR Docker artifact"
-on:
-  push:
-    branches:
-      # NB: Default-branch doesn't work, despite Github's documentation.
-      #- $default-branch
-      - master
-env:
-  IMAGE_NAME: storyteller-rust
-
-jobs:
-  # Push image to GitHub Packages.
-  # See also https://docs.docker.com/docker-hub/builds/
-  # See also https://depot.dev/blog/docker-layer-caching-in-github-actions
-  # See also https://evilmartians.com/chronicles/build-images-on-github-actions-with-docker-layer-caching
-  build-docker-image:
-    name: "Build Docker Image"
-    #runs-on: ubuntu-22.04 # runs-on: ubuntu-latest
-    #runs-on: github-ubuntu-16core # NB: Premium GitHub runner with more cores
-    runs-on: github-ubuntu-64core # NB: Premium GitHub runner with more cores
-    if: github.event_name == 'push'
-    permissions:
-      packages: write
-      contents: read
-
-    steps:
-      - uses: actions/checkout@v3
-
-      # - uses: docker/setup-buildx-action@v1
-      #   name: Set up Docker Buildx
-
-      - uses: docker/login-action@v1
-        name: Login to GitHub Container Registry
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }} # username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Calculate short SHA
-        id: vars
-        # echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-        run: |
-          echo "sha_short=$(echo ${{ github.sha }} | cut -c1-12)" >> $GITHUB_OUTPUT
-
-      - name: Emit short SHA
-        run: |
-          echo "Full SHA: ${{ github.sha }}"
-          echo "Short SHA: ${{ steps.vars.outputs.sha_short }}"
-
-      #- uses: actions/cache@v2
-      #  name: Cache Docker layers
-      #  with:
-      #    path: /tmp/.buildx-cache
-      #    key: ${{ runner.os }}-buildx-${{ github.sha }}
-      #    restore-keys: |
-      #      ${{ runner.os }}-buildx-
-
-      # Docs for this action are at https://github.com/marketplace/actions/build-docker-images-using-cache
-      - uses: whoan/docker-build-with-cache-action@v6
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }} # username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-          image_name: storytold/storyteller-rust
-          dockerfile: service_cpu.Dockerfile
-          # NB: The `git_push_tag: true` option does not work as expected, so we put the sha in `image_tag`:
-          # https://github.com/whoan/docker-build-with-cache-action/issues/72
-          image_tag: latest,${{ github.sha }},${{ steps.vars.outputs.sha_short }}
-          # NB: Inject the Git SHA into the container so the servers can emit their own version:
-          build_extra_args: "--build-arg=GIT_SHA=${{ github.sha }}"
-
-      # - name: Build image
-      #   run: |
-      #     SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
-      #     docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" --build-arg GIT_SHA=${SHORT_SHA}
-
-      # - name: Push image to GitHub Container Registry
-      #   run: |
-      #     IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
-
-      #     # Change all uppercase to lowercase
-      #     IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
-
-      #     # Strip git ref prefix from version
-      #     VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-
-      #     # Strip "v" prefix from tag name
-      #     [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
-
-      #     # Use Docker `latest` tag convention
-      #     [ "$VERSION" == "$default-branch" ] && VERSION=latest
-
-      #     echo IMAGE_ID=$IMAGE_ID
-      #     echo VERSION=$VERSION
-
-      #     docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
-      #     docker push $IMAGE_ID:$VERSION
-
-      #     SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
-
-      #     docker tag $IMAGE_NAME $IMAGE_ID:$SHORT_SHA
-      #     docker push $IMAGE_ID:$SHORT_SHA
-
-      - name: Cleanup / Debug
-        # This job will run even on failures to help with debugging.
-        if: always()
-        run: |
-          echo "Report at final github action step:"
-          echo "Disk usage:"
-          pwd
-          du -hsc * | sort -hr

.github/workflows/docker-publish-gpu.yml

@@ -1,115 +0,0 @@
-# Adapted from
-# https://github.com/actions/starter-workflows/ci/docker-publish.yml
-# If using base images, don't forget to add READ access, eg. from a package settings page like:
-# https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access
-name: "[GPU] Build GHCR Docker artifact"
-on:
-  push:
-    branches:
-      # NB: Default-branch doesn't work, despite Github's documentation.
-      #- $default-branch
-      - master
-env:
-  IMAGE_NAME: storyteller-rust-gpu
-
-jobs:
-  # Push image to GitHub Packages.
-  # See also https://docs.docker.com/docker-hub/builds/
-  # See also https://depot.dev/blog/docker-layer-caching-in-github-actions
-  # See also https://evilmartians.com/chronicles/build-images-on-github-actions-with-docker-layer-caching
-  build-docker-image:
-    name: "Build Docker Image"
-    #runs-on: ubuntu-22.04 # runs-on: ubuntu-latest
-    #runs-on: github-ubuntu-16core # NB: Premium GitHub runner with more cores
-    runs-on: github-ubuntu-64core # NB: Premium GitHub runner with more cores
-    if: github.event_name == 'push'
-    permissions:
-      packages: write
-      contents: read
-
-    steps:
-      - uses: actions/checkout@v3
-
-      # - uses: docker/setup-buildx-action@v1
-      #   name: Set up Docker Buildx
-
-      - uses: docker/login-action@v1
-        name: Login to GitHub Container Registry
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }} # username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Calculate short SHA
-        id: vars
-        # echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-        run: |
-          echo "sha_short=$(echo ${{ github.sha }} | cut -c1-12)" >> $GITHUB_OUTPUT
-
-      - name: Emit short SHA
-        run: |
-          echo "Full SHA: ${{ github.sha }}"
-          echo "Short SHA: ${{ steps.vars.outputs.sha_short }}"
-
-      #- uses: actions/cache@v2
-      #  name: Cache Docker layers
-      #  with:
-      #    path: /tmp/.buildx-cache
-      #    key: ${{ runner.os }}-buildx-${{ github.sha }}
-      #    restore-keys: |
-      #      ${{ runner.os }}-buildx-
-
-      # Docs for this action are at https://github.com/marketplace/actions/build-docker-images-using-cache
-      - uses: whoan/docker-build-with-cache-action@v6
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }} # username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-          image_name: storytold/storyteller-rust-gpu
-          dockerfile: service_gpu.Dockerfile
-          # NB: The `git_push_tag: true` option does not work as expected, so we put the sha in `image_tag`:
-          # https://github.com/whoan/docker-build-with-cache-action/issues/72
-          image_tag: latest,${{ github.sha }},${{ steps.vars.outputs.sha_short }}
-          # NB: Inject the Git SHA into the container so the servers can emit their own version:
-          build_extra_args: "--build-arg=GIT_SHA=${{ github.sha }}"
-
-      # - name: Build image
-      #   run: |
-      #     SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
-      #     docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" --build-arg GIT_SHA=${SHORT_SHA}
-
-      # - name: Push image to GitHub Container Registry
-      #   run: |
-      #     IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
-
-      #     # Change all uppercase to lowercase
-      #     IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
-
-      #     # Strip git ref prefix from version
-      #     VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-
-      #     # Strip "v" prefix from tag name
-      #     [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
-
-      #     # Use Docker `latest` tag convention
-      #     [ "$VERSION" == "$default-branch" ] && VERSION=latest
-
-      #     echo IMAGE_ID=$IMAGE_ID
-      #     echo VERSION=$VERSION
-
-      #     docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
-      #     docker push $IMAGE_ID:$VERSION
-
-      #     SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
-
-      #     docker tag $IMAGE_NAME $IMAGE_ID:$SHORT_SHA
-      #     docker push $IMAGE_ID:$SHORT_SHA
-
-      - name: Cleanup / Debug
-        # This job will run even on failures to help with debugging.
-        if: always()
-        run: |
-          echo "Report at final github action step:"
-          echo "Disk usage:"
-          pwd
-          du -hsc * | sort -hr

.github/workflows/docker-publish.yml

@@ -0,0 +1,67 @@
+# Adapted from
+# https://github.com/actions/starter-workflows/ci/docker-publish.yml
+# If using base images, don't forget to add READ access, eg. from a package settings page like:
+# https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access
+name: Publish Docker Container to GHCR
+on:
+  push:
+    branches:
+      # NB: Default-branch doesn't work, despite Github's documentation.
+      #- $default-branch
+      - master
+env:
+  IMAGE_NAME: storyteller-rust
+
+jobs:
+  # Push image to GitHub Packages.
+  # See also https://docs.docker.com/docker-hub/builds/
+  push:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          #username: ${{ github.repository_owner }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }} 
+
+      - name: Build image
+        run: |
+          SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
+          docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" --build-arg GIT_SHA=${SHORT_SHA}
+
+      - name: Push image to GitHub Container Registry
+        run: |
+          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+
+          # Change all uppercase to lowercase
+          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+
+          # Strip git ref prefix from version
+          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+
+          # Strip "v" prefix from tag name
+          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+
+          # Use Docker `latest` tag convention
+          [ "$VERSION" == "$default-branch" ] && VERSION=latest
+
+          echo IMAGE_ID=$IMAGE_ID
+          echo VERSION=$VERSION
+
+          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION
+
+          SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
+
+          docker tag $IMAGE_NAME $IMAGE_ID:$SHORT_SHA
+          docker push $IMAGE_ID:$SHORT_SHA
+

.github/workflows_inactive/ci.yml

@@ -1,52 +0,0 @@
-name: CI
-on: [push, pull_request]
-env: 
-  MYSQL_URL: mysql://root:root@localhost/storyteller
-  DATABASE_URL: mysql://root:root@localhost/storyteller
-  REDIS_0_URL: redis://redisd@localhost/0
-  REDIS_1_URL: redis://redisd@localhost/1
-  DEBIAN_FRONTEND: noninteractive
-
-jobs:
-  setup_build_test:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-      - name: Cache cargo registry
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
-      - name: Setup mysql
-        run: |
-          sudo systemctl start mysql
-          sudo apt-get update
-          sudo apt-get -y install mysql-client libmysqlclient-dev
-          mysql -e "create user 'storyteller'@'localhost' IDENTIFIED BY 'password';" -uroot -proot 
-          mysql -e "create database storyteller; grant all on storyteller.* to 'storyteller'@'localhost';" -uroot -proot
-      - name: Setup Redis
-        run: |
-          sudo apt-get -y install redis-server
-          sudo service redis-server start
-      - name: Setup Rust
-        run: |
-          rustup set profile minimal
-          rustup show
-          rustup install 1.66
-          rustup default 1.66
-          rustup show
-          cargo install sqlx-cli --features openssl-vendored
-          cargo install diesel_cli
-      - name: Run database migrations
-        run: |
-          # nothing to see here ;)
-          set +e +o pipefail
-          diesel migration run
-          set -e -o pipefail
-      - name: Build 
-        run: cargo build --release
-      - name: Test
-        run: cargo test --release

.gitignore

@@ -1,59 +1,22 @@
-# Secrets are BAD!
+*.swo
+*.swp
 *secret*sh
+*~
 .env
-.env-aichatbot-secrets
 .env-secrets
+.idea/
+__pycache__/
+_ignored_schema.rs
 db.secrets
 db_secrets.toml
+models/Wav2Lip/python/
+models/tacotron2/python/
 production_database_secret_connect.sh
 production_database_secret_query.sh
 production_database_voice_clone_report_download.sh
 secrets.toml
 secrets.yaml
-_develop/tools/.secrets
-_develop/tools/db_secrets.toml
-twitch_secrets.toml
-crates/service/web/storyteller_web/config/storyteller-web.development-secrets.env
-
-# New env secrets (per-app)
-*-secrets.env
-
-# Rust build directory
 target/
-crates/lib/database_queries/target/
-crates/lib/sqlite_queries/target/
-
-# Vim noise
-*.swo
-*.swp
-*~
-
-# IntelliJ/CLion noise
-.idea/
-
-# Python noise
-__pycache__/
-
-# We use diesel for DB migrations only. We don't care about its codgen.
-_ignored_schema.rs
-
-# AI Chat Sidecar has its own schema managed by diesel, but we don't care for diesel codegen.
-crates/programs/aichatbot-sidecar/_ignored_schema.rs
-
-# AI Chat sidecar's SQlite database shouldn't be checked in
-crates/programs/aichatbot-sidecar/database.db
-
-# Runtime data for the AI chatbot
-runtime_data/
-
-# Old Python virtual environments
-models/Wav2Lip/python/
-models/tacotron2/python/
-
-# Stuff I want to keep around, but not commit
-scraps/
-
-# Windows build files
-# These should be built by the platform
-windows/sqlite/sqlite3.exp
-windows/sqlite/sqlite3.lib
+tools/.secrets
+tools/db_secrets.toml
+twitch_secrets.toml

.idea/dictionaries/bt.xml

@@ -1,45 +1,18 @@
 <component name="ProjectDictionaryState">
+  <!--
+  Despite spellcheck dictionaries being created per-user, IntelliJ loads all of them, per:
+  https://stackoverflow.com/a/28151692
+  -->
   <dictionary name="bt">
     <words>
       <w>arpabet</w>
       <w>backfill</w>
-      <w>cashapp</w>
       <w>cheermote</w>
-      <w>cidrs</w>
-      <w>crockford</w>
-      <w>culted</w>
-      <w>dalle</w>
-      <w>demuxer</w>
-      <w>dereferencing</w>
-      <w>downvote</w>
       <w>fakeyou</w>
-      <w>ffprobe</w>
-      <w>greenlight</w>
-      <w>greenlighting</w>
-      <w>greenlit</w>
-      <w>hanashi</w>
       <w>hifigan</w>
-      <w>hparams</w>
-      <w>huggingface</w>
-      <w>lipsync</w>
-      <w>pathbuf</w>
-      <w>replayable</w>
-      <w>retryably</w>
       <w>scrollback</w>
-      <w>softvc</w>
-      <w>sql's</w>
-      <w>symphonia</w>
-      <w>tacotron</w>
-      <w>techcrunch</w>
-      <w>transcoded</w>
-      <w>transcodings</w>
-      <w>unpermitted</w>
-      <w>upserting</w>
-      <w>vits</w>
-      <w>vocoded</w>
       <w>vocoder</w>
       <w>vocoders</w>
-      <w>vocodes</w>
     </words>
   </dictionary>
 </component>

Cargo.toml

@@ -1,74 +1,29 @@
 [workspace]
 
 members = [
-  # Types: common strongly-typed tokens, ids, payloads, etc.
-  # Also common cloud buckets and other "strongly typed" configs.
-  "crates/_types/buckets",
-  "crates/_types/enums",
-  "crates/_types/reusable_types", # TODO(bt, 2022-12-21): Deprecate
-  "crates/_types/tokens",
-
-  # CLI tools for operations, etc.
-  "crates/cli/database_backup",
-  "crates/cli/database_cleanup",
-  "crates/cli/scrape_test",
-
   # Shared libraries
-  "crates/lib/actix_helpers",
-  "crates/lib/api_clients/cloud_storage",
-  #"crates/lib/api_clients/fakeyou_client",
-  "crates/lib/api_clients/google_drive_common",
-  "crates/lib/api_clients/twitch_common",
-  "crates/lib/bootstrap",
-  "crates/lib/caching/memory_caching",
-  "crates/lib/caching/redis_caching",
-  "crates/lib/concurrency",
-  "crates/lib/crockford",
-  "crates/lib/datetimes",
-  "crates/lib/deprecated/config", # TODO(bt, 2023-02-13): Deprecate or repurpose
-  "crates/lib/deprecated/container_common", # TODO(bt, 2023-02-13): Pull apart and rehome pieces
-  "crates/lib/deprecated/http_server_common", # TODO(bt, 2023-03-20): Move useful pieces to actix_helpers
-  "crates/lib/envvar",
-  "crates/lib/errors",
-  "crates/lib/files/archives",
-  "crates/lib/files/filesys",
-  "crates/lib/files/hashing",
-  "crates/lib/files/media",
-  "crates/lib/files/mimetypes",
-  "crates/lib/idempotency",
+  "crates/lib/config",
+  "crates/lib/container_common",
+  "crates/lib/database_queries",
+  "crates/lib/google_drive_common",
+  "crates/lib/http_server_common",
   "crates/lib/jobs_common",
-  "crates/lib/mysql_queries",
+  "crates/lib/memory_caching",
   "crates/lib/redis_common",
-  "crates/lib/server_environment",
-  #"crates/lib/sqlite_queries",
-  "crates/lib/sqlx_mysql_helpers",
-  "crates/lib/subprocess_common",
+  "crates/lib/reusable_types",
+  "crates/lib/storage_buckets_common",
   "crates/lib/tts_common",
-  "crates/lib/url_config",
+  "crates/lib/twitch_common",
   "crates/lib/user_input_common",
-  "crates/lib/web_scrapers",
 
-  # Programs: Desktop apps
-  #"crates/programs/aichatbot-sidecar",
-
-  # Services: API servers, jobs, workers, etc.
-  "crates/service/job/analytics_job",
-  "crates/service/job/download_job",
-  "crates/service/job/inference_job",
-  "crates/service/job/tts_download_job",
-  "crates/service/job/tts_inference_job",
-  "crates/service/job/w2l_download_job",
-  "crates/service/job/w2l_inference_job",
-  "crates/service/other/twitch_pubsub_subscriber",
-  "crates/service/web/dummy_service",
-  "crates/service/web/storyteller_web",
-  "crates/service/web/websocket_gateway",
-
-  # Service Plugins: complete subsystems that can be plugged into services (users, billing, etc.)
-  "crates/service/plugins/billing",
-  "crates/service/plugins/inference_workloads",
-  "crates/service/plugins/users",
-
-  # Vendored code
-  "crates/vendor/newrelic-telemetry-sdk-rust",
+  # Application 
+  "crates/service/generic_download_job",
+  "crates/service/storyteller_web",
+  "crates/service/tts_download_job",
+  "crates/service/tts_inference_job",
+  "crates/service/twitch_pubsub_subscriber",
+  "crates/service/w2l_download_job",
+  "crates/service/w2l_inference_job",
+  "crates/service/websocket_gateway",
 ]
+

Cranky.toml

@@ -1,58 +0,0 @@
-# Persist clippy check configurations:
-# https://github.com/ericseppanen/cargo-cranky 
-# Clippy doesn't allow this (yet):
-# https://github.com/rust-lang/rust-clippy/issues/6625
-deny = [
-  # Rust
-  "unsafe_code", # We may remove this if needed
-
-  # Clippy - "serious" lints
-  "clippy::borrow_interior_mutable_const", # Seems dangerous to allow this. Eg. `const F = Lazy::new()`;
-
-  # Clippy - bad math
-  #"clippy::cast_possible_truncation", # Seems dangerous, eg f64 to u64.
-  #"clippy::cast_possible_wrap",
-
-  # Clippy - pointer safety / readability / correctness
-  "clippy::as_ptr_cast_mut",
-  "clippy::borrow_as_ptr",
-  "clippy::borrow_deref_ref",
-
-  # Clippy - performance
-  "clippy::cmp_owned",
-  "clippy::useless_vec",
-  "clippy::format_in_format_args",
-  #"clippy::redundant_clone",
-  #"clippy::unnecessary_to_owned",
-  #"clippy::vec_init_then_push",
-
-  # Clippy - "ergonomic" / "usage" lints
-  "clippy::option_option", # Lint: Don't ever Option<Option<T>>! Enums instead.
-  "clippy::zero_sized_map_values", # Lint: use a HashSet instead of HashMap<T, ()>.
-
-  # Clippy - "aesthetic" / "style" lints
-  "clippy::clone_double_ref", # Lint: feels safer/hygienic to disallow this, but I'm not strongly opposed.
-  "clippy::double_comparisons", # Lint: readability
-  "clippy::empty_structs_with_brackets", # Lint: {} feels less aesthetic than ;
-  "clippy::implicit_clone", # Lint: Recommendation here seems more idiomatic.
-  "clippy::needless_return", # Lint: not the worst offender, but let's try to keep unnecessary returns out.
-  "clippy::redundant_field_names", # Lint: Duplicate names is a burden when reading: you wonder why they're dupes.
-  "clippy::redundant_static_lifetimes", # Lint: feels more modern to elide static lifetimes
-
-  # Possible:
-  # "clippy::ptr_arg",
-  # "clippy::too_many_lines" (note: this is configurable)
-]
-
-warn = [
-]
-
-allow = [
-  # Rust
-  "deprecated", # Other systems can handle deprecated code instead of the linter
-  "missing_docs", # Not everything needs documentation. Too much documentation is bad. Literate code.
-
-  # Clippy
-  "clippy::if_same_then_else", # There are cases this is clearer! Branches may have notes / considerations.
-  "clippy::single_char_pattern", # This is like punishing trailing commas. Chars are dumb.
-]

Dockerfile

@@ -0,0 +1,187 @@
+# ==================== Python Build Base ====================
+
+FROM ghcr.io/storytold/docker-base-images-nvidia-cuda-experimental:080f96bc7087 as pybuild-base
+
+# The downloader needs youtube-dl as a python library
+# NB: No longer maintained: https://news.ycombinator.com/item?id=28319624
+#RUN pip3 install --upgrade youtube-dl
+
+# Used by the downloader script.
+# And it looks like youtube-dl is dead...
+# Per the README, it looks like pip will always pull the latest version.
+# https://github.com/yt-dlp/yt-dlp#update
+RUN python3 -m pip install --upgrade yt-dlp
+
+# ==================== Python Build Step 2: Wav2Lip Requirements ====================
+
+FROM pybuild-base as pybuild-requirements
+
+WORKDIR /
+
+COPY models/Wav2Lip ./models/Wav2Lip
+WORKDIR models/Wav2Lip
+
+# NB: We need Python3.6
+RUN python3.6 -m venv python
+#RUN virtualenv -p /usr/bin/python3.6  ... wait
+RUN . python/bin/activate \
+  && pip install --upgrade pip \
+  && pip install -r requirements.txt \
+  && deactivate
+
+WORKDIR /
+
+COPY models/tts ./models/tts
+WORKDIR models/tts
+
+# NB: Not sure if we need Python3.6 for Tacotron2, but Python3.8 gave me trouble.
+# NB: Setuptools fix: https://github.com/tensorflow/tensorflow/issues/34302#issuecomment-554450289
+RUN python3.6 -m venv python
+RUN . python/bin/activate \
+  && pip install --upgrade pip setuptools \
+  && pip install -r requirements-tacotron-python36.txt \
+  && deactivate
+
+# ==================== Rust Build Base ====================
+
+# Custom base image
+# Make sure to add this repository so it has read acces to the base image:
+# https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access
+FROM ghcr.io/storytold/docker-base-images-rust-ssl:latest as rust-build
+WORKDIR /tmp
+
+COPY Cargo.lock . 
+COPY Cargo.toml .
+COPY sqlx-data.json .
+COPY crates/ ./crates
+COPY db/ ./db
+
+RUN $HOME/.cargo/bin/cargo fetch
+
+ARG GIT_SHA
+RUN echo -n ${GIT_SHA} > GIT_SHA
+
+# Run all of the tests
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo test
+
+# Build all the binaries.
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin storyteller-web
+
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin tts-download-job
+
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin w2l-download-job
+
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin generic-download-job
+
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin tts-inference-job
+
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin w2l-inference-job
+
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin websocket-gateway
+
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin twitch-pubsub-subscriber
+
+# Final image
+#  FROM ubuntu:xenial
+FROM pybuild-requirements as final
+
+WORKDIR /
+
+COPY scripts/ ./scripts
+
+# See: https://github.com/opencontainers/image-spec/blob/master/annotations.md
+LABEL org.opencontainers.image.authors='bt@brand.io, echelon@gmail.com'
+LABEL org.opencontainers.image.url='https://github.com/storytold/storyteller-web'
+LABEL org.opencontainers.image.documentation='https://github.com/storytold/storyteller-web'
+LABEL org.opencontainers.image.source='https://github.com/storytold/storyteller-web'
+
+# NB(echelon,2022-05-31): NVIDIA revoked some of their keys.
+# A better long term fix is to remove these dependencies outright and not have ML deps in this repo.
+# https://developer.nvidia.com/blog/updating-the-cuda-linux-gpg-repository-key/
+# https://forums.developer.nvidia.com/t/invalid-public-key-for-cuda-apt-repository/212901/11
+RUN apt-key del 7fa2af80
+
+# NB: Fix for 'E: Conflicting values set for option Signed-By regarding source'
+# NB: We need to remove all sources
+RUN rm $(find /etc/apt | grep cuda)
+
+#RUN wget https://developer.download.nvidia.com/compute/cuda/repos/$distro/$arch/cuda-keyring_1.0-1_all.deb
+RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb
+RUN dpkg -i cuda-keyring_1.0-1_all.deb
+
+# NB: Comment this out for non-debug images
+# TODO: Figure out how this is done elsewhere with just the static binaries
+# Others: `mysql-client`
+RUN apt-get update \
+    && apt-get install -y \
+        curl \
+        python3-pip \
+        python3 \
+        wget
+
+RUN python3 --version
+
+# TODO: Use venv to do this instead.
+RUN pip3 install gdown
+
+# Copy all the binaries.
+COPY --from=rust-build /tmp/target/release/storyteller-web /
+COPY --from=rust-build /tmp/target/release/tts-download-job /
+COPY --from=rust-build /tmp/target/release/tts-inference-job /
+COPY --from=rust-build /tmp/target/release/w2l-download-job /
+COPY --from=rust-build /tmp/target/release/w2l-inference-job /
+COPY --from=rust-build /tmp/target/release/websocket-gateway /
+COPY --from=rust-build /tmp/target/release/twitch-pubsub-subscriber /
+COPY --from=rust-build /tmp/target/release/generic-download-job /
+COPY --from=rust-build /tmp/GIT_SHA /
+
+# SSL certs are required for crypto
+COPY --from=rust-build /etc/ssl /etc/ssl
+
+# Required dynamically linked libraries
+COPY --from=rust-build /lib/x86_64-linux-gnu/libssl.*             /lib/x86_64-linux-gnu/
+COPY --from=rust-build /lib/x86_64-linux-gnu/libcrypto.*          /lib/x86_64-linux-gnu/
+
+# Make sure all the links resolve
+RUN ldd storyteller-web
+
+# Without a .env file, Rust crashes "mysteriously" (ugh)
+RUN touch .env
+RUN touch .env-secrets
+
+EXPOSE 8080
+CMD LD_LIBRARY_PATH=/usr/lib /storyteller-web

README.md

@@ -1,49 +1,77 @@
 storyteller-rust
 ================
 
-Our Rust monorepo, which serves both our API surface area and our downloadable client applications.
+Our Rust monorepo, which services our 
+API surface area. 
+
+Application overview
+--------------------
+
+* [FakeYou.com](https://fakeyou.com) is powered by:
+  * HTTP APIs:
+    * `storyteller-web` - HTTP + database monolith for FakeYou
+  * Asynchronous Jobs:
+    * `tts-download-job` - Async download of models
+    * `tts-inference-job` - Async TTS inference
+    * `w2l-download-job` - Async download of videos and images
+    * `w2l-inference-job` - Async W2L inference
+
+* [StorytellerStream](https://storyteller.stream) is powered by:
+  * HTTP APIs:
+    * `storyteller-web` - HTTP + database monolith for FakeYou
+  * Websocket events:
+    * `websocket-gateway` - Websocket to pull down Twitch events.
+  * Twitch API integration:
+    * `twitch-pubsub-subscriber` - Dynamically subscribes to Twitch PubSub for "active" users
+
+Schema and API notes
+--------------------
+
+* [Redis Schema Notes README](_docs/redis_schema.md)
+
+Local development
+-----------------
+
+[Local development setup README](_docs/local_development_setup.md)
+
+Production and Deployment
+-------------------------
+
+[Production builds and deployment README](_docs/production_builds_and_deployment.md)
+
+Development notes
+-----------------
+
+[Development notes README](_docs/development_notes.md)
+
+Dump a MySQL report
+-------------------
 
 ```
-                 ."-,.__
-                 `.     `.  ,
-              .--'  .._,'"-' `.
-             .    .'         `'
-             `.   /          ,'
-               `  '--.   ,-"'
-                `"`   |  \
-                   -. \, |
-                    `--Y.'      ___.
-                         \     L._, \
-               _.,        `.   <  <\                _
-             ,' '           `, `.   | \            ( `
-          ../, `.            `  |    .\`.           \ \_
-         ,' ,..  .           _.,'    ||\l            )  '".
-        , ,'   \           ,'.-.`-._,'  |           .  _._`.
-      ,' /      \ \        `' ' `--/   | \          / /   ..\
-    .'  /        \ .         |\__ - _ ,'` `        / /     `.`.
-    |  '          ..         `-...-"  |  `-'      / /        . `.
-    | /           |L__           |    |          / /          `. `.
-   , /            .   .          |    |         / /             ` `
-  / /          ,. ,`._ `-_       |    |  _   ,-' /               ` \
- / .           \"`_/. `-_ \_,.  ,'    +-' `-'  _,        ..,-.    \`.
-.  '         .-f    ,'   `    '.       \__.---'     _   .'   '     \ \
-' /          `.'    l     .' /          \..      ,_|/   `.  ,'`     L`
-|'      _.-""` `.    \ _,'  `            \ `.___`.'"`-.  , |   |    | \
-||    ,'      `. `.   '       _,...._        `  |    `/ '  |   '     .|
-||  ,'          `. ;.,.---' ,'       `.   `.. `-'  .-' /_ .'    ;_   ||
-|| '              V      / /           `   | `   ,'   ,' '.    !  `. ||
-||/            _,-------7 '              . |  `-'    l         /    `||
-. |          ,' .-   ,' ||               | .-.        `.      .'     ||
- `'        ,'    `".'    |               |    `.        '. -.'       `'
-          /      ,'      |               |,'    \-.._,.'/'
-          .     /        .               .       \    .''
-        .`.    |         `.             /         :_,'.'
-          \ `...\   _     ,'-.        .'         /_.-'                       S T O R Y T E L L E R
-           `-.__ `,  `'   .  _.>----''.  _  __  /                        Turning memes into dreams
-                .'        /"'          |  "'   '_                      One Rust Monorepo at a time
-               /_|.-'\ ,".             '.'`__'-( \
-                 / ,"'"\,'               `/  `-.|" 
+mysql -u storyteller \
+  -pPasswordHere \
+  -h IP_HERE \
+  -P 3306 \
+  -D storyteller \
+  -B \
+  -e "select * from voice_clone_requests;" | sed "s/'/\'/;s/\t/\",\"/g;s/^/\"/;s/$/\"/;s/\n//g" > voice_clone_report.csv
 ```
 
-- [documentation](./_develop/docs)
-- [tools, misc](./_develop)
+TODO
+----
+
+* Revise this README to be more useful.
+
+Notes / TODOs:
+
+* Examples for good Actix+Sqlx Tests:
+  https://stackoverflow.com/questions/65370752/how-do-i-create-an-actix-web-server-that-accepts-both-sqlx-database-pools-and-tr
+
+* Actix/sqlx runtime compat:
+  https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734
+  
+* Redis caching of more endpoints
+
+* Jobs for analytics queries (#uses per model, etc)
+* Jobs for ingesting Discord, Twitter, Reddit RPAN chat, Twitch chat (not PubSub), etc.
+

_develop/docs/README.md

@@ -1,52 +0,0 @@
-README
-======
-
-This directory contains a collection of (hopefully organized and up to date) documentation on 
-running the various backend pieces, development setup, production maintenance, etc. Check out 
-all the markdown documents in this directory tree for information organized by topic.
-
-Development / Setup
--------------------
-
-- [Most of the general documents for cross-cutting setup of FakeYou.com live here.](./local_development)
-- Some per-application documents (such as for `aichatbot-sidecar`) live as READMEs under that particular 
-  application's code directory, eg. `/crates/{appCategory}/{appName}/README.md` (These are irrelevant 
-  for most folks.)
-
-Web API Overview
-----------------
-
-### FakeYou.com
-
-* [FakeYou.com](https://fakeyou.com) is powered by the following applications:
-    * HTTP APIs:
-        * `storyteller-web` - HTTP API + database monolith for FakeYou
-    * Asynchronous Jobs:
-        * `download-job` - Handle async download of ML models (RVC, SVC, etc.)
-        * `inference-job` - Run async ML model inference (RVC, SVC, SadTalker, etc.) 
-                            This will handle most model inference going forward.
-        * `tts-download-job` - (*deprecated*) Async download of Tacotron2 models only
-        * `tts-inference-job` - (*deprecated*) Async TTS inference of Tacotron2 models only
-        * `w2l-download-job` - (*deprecated*) Async download of videos and images
-        * `w2l-inference-job` - (*deprecated*) Async W2L inference
-
-### PowerStream
-
-* [PowerStream](https://power.stream) (deprecated) is powered by the following applications:
-    * HTTP APIs:
-        * `storyteller-web` - HTTP + database monolith for FakeYou ***and*** PowerStream.
-    * Twitch API integration:
-        * `twitch-pubsub-subscriber` - Dynamically subscribes to Twitch PubSub for "active" users
-    * Websocket events:
-        * `websocket-gateway` - Websocket for users' browsers to pull down enriched Twitch events.
-
-TODO
-----
-
-Notes / TODOs:
-
-* Examples for good Actix+Sqlx Tests:
-  https://stackoverflow.com/questions/65370752/how-do-i-create-an-actix-web-server-that-accepts-both-sqlx-database-pools-and-tr
-
-* Actix/sqlx runtime compat:
-  https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734

_develop/docs/cheat_sheet/migrations.md

@@ -1,43 +0,0 @@
-Migrations guide
-================
-
-## Diesel Migrations
-
-To run pending migrations, execute the following:
-
-```bash
-diesel migration run
-```
-
-To revert the last 3 migrations:
-
-```bash
-diesel migration revert -n 3
-```
-
-To reset the entire database (drop, migrate), run:
-
-```bash
-diesel database reset
-```
-
-## Server Query Codegen
-
-We use SQLx instead of Diesel in the production server. It's typesafe
-SQL instead of an ORM like Diesel.
-
-SQLx connects to a database to derive type information, but obviously
-cannot do this for builds in CI. In order to cache the types, we build
-and check in a cache file (necessary for builds):
-
-```bash
-SQLX_OFFLINE=true cargo sqlx prepare
-```
-
-Now that we have multiple binaries, it's required to include all the queries in the main
-binary so we can generate the cached queries as a single target. That's then executed
-with:
-
-```bash
-SQLX_OFFLINE=true cargo sqlx prepare -- --bin storyteller-web
-```

_develop/docs/local_development/general_setup.md

@@ -1,207 +0,0 @@
-FakeYou Development Setup (bare metal)
-======================================
-
-This document describes how to run the FakeYou infrastructure on your development 
-machine's bare metal (no containers).
-
-Setup instructions and development were tested and known to work on the following
-platforms:
-
-- Mac OS (Apple M2 silicon)
-- Ubuntu 22.04
-
-The applications (binary targets) you might be interested in running include, but 
-are not limited to:
-
-- `storyteller-web`, the HTTP API server
-- `download-job`, which downloads models from the web and stores them in GCS
-- `inference-job`, which downloads models from GCS and runs inference with them
-
-Setup Instructions
-------------------
-
-### (1) Install Rust
-
-Install Rust [using this guide](https://www.rust-lang.org/learn/get-started). If it asks, 
-you'll want "stable" Rust, not "nightly" Rust. If it doesn't ask, it defaults to "stable".
-
-### (2) Install the necessary libraries
-
-Mac: (Lost the list, but it should match Linux. Install Homebrew.)
-
-Ubuntu 22.04:
-
-```bash
-sudo apt install \ 
-  jq \
-  libmysqlclient-dev \
-  libsqlite3-dev \
-  pkgconf
-```
-
-(Note: jq is for combining mysql codegen outputs; pkgconf is if using openssl instead of rustls.)
-
-### (3) Install MySQL server
-
-Mac:
-
-```bash
-brew install mysql
-```
-
-If `mysql -uroot` fails, reboot the machine:
-
-```bash
-sudo reboot now
-```
-
-Ubuntu 22.04:
-
-```bash
-sudo apt install mysql-server
-```
-
-
-### (4) Install a `storyteller` user and table in MySQL
-
-Connect to mysql:
-
-```bash
-# If the following command asks for a password, the password is typically "root"
-sudo mysql -u root -p
-```
-
-Once in MySQL, run the following:
-
-```mysql
-use mysql;
-CREATE DATABASE storyteller;
-CREATE USER 'storyteller'@'localhost' IDENTIFIED BY 'password';
-GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';
-```
-
-Then verify access with `./dev_mysql_connect.sh`
-
-### (5) Install Diesel CLI (for MySQL migrations):
-
-```bash
-cargo install diesel_cli \
-  --no-default-features \
-  --features mysql,sqlite
-```
-
-As of this writing, Mac [has some issues with Diesel CLI](https://github.com/diesel-rs/diesel/issues/2605)
-and requires a few extra dependencies to be installed:
-
-```bash
-brew install libpq
-```
-
-### (6) Run the pending database migrations:
-
-```bash
-diesel migration run
-```
-
-You might get a scary message about `"Encountered unknown type for Mysql: enum"` -- you can ignore this 
-error (see below).
-
-### (7) (optional) Install the SQLx CLI (if doing database development)
-
-```bash
-cargo install sqlx-cli --no-default-features --features rustls,mysql,sqlite
-```
-
-We'll be using diesel to manage the migrations, but sqlx within the app to actually perform queries.
-Diesel is an immature ORM, which isn't a good tech bet, so we use sqlx as at-compile-time
-typesafe SQL.
-
-### (8) Install hosts file:
-
-(This should be the same on Linux and Mac, but might differ for Windows.)
-
-If you're developing against the frontend, it'll target development domains (eg. `dev.fakeyou.com`) instead 
-of `127.0.0.1` or `localhost`. You can make your machine route domains to localhost by editing your hosts 
-file (located at `/etc/hosts`) to include the following configuration lines:
-
-```
-127.0.0.1    dev.fakeyou.com
-127.0.0.1    api.dev.fakeyou.com
-127.0.0.1    devproxy.fakeyou.com
-
-127.0.0.1    dev.storyteller.ai
-127.0.0.1    api.dev.storyteller.ai
-127.0.0.1    devproxy.storyteller.ai
-```
-
-### (9a) Install Redis
-
-```
-sudo apt install redis
-```
-
-### (9b) Run one or more applications:
-DO NOT CHECK THESE TWO FILES INTO THE REPO
-Ask Brandon for the .env-secrets and .storteller-web.development-secrets.env and place it in the root of the folder.
-
-To start the HTTP API server,
-
-```bash
-cargo run --bin storyteller-web
-```
-Note that this compiles and runs the "development" binary. It's faster and easier to debug than the
-optimized "production" build. To build a fully optimized production release,
-run `cargo build --release --bin storyteller-web` . Note that this will take much longer.
-
-DO NOT CHECK THESE TWO FILES INTO THE REPO
-
-Ask Brandon for the .download-job.development-secrets.env and place it in
-crates/service/job/download_job/config
-
-Ask Brandon for the .inference-job.development-secrets.env and place it in
-crates/service/job/inference_job/config
-
-To download some ML models, run:
-
-```bash
-cargo run --bin download-job
-cargo run --bin tts-download-job
-```
-
-If you want to run both the HTTP API and the jobs, you'll need to run both processes in different
-terminals or tmux sessions.
-
-Note also that the configurations pointing to the ML monorepo must be set up for each application.
-Additionally, some development secrets may be needed (ask the team to share).
-
-To execute ML inference, run:
-
-```bash
-cargo run --bin inference-job
-cargo run --bin tts-inference-job
-```
-
-Again, note that the configurations pointing to the ML monorepo must be set up for each application.
-Additionally, some development secrets may be needed (ask the team to share).
-
-Solutions to Common Setup Problems 
------------------------------------
-
-### [Fix] Sqlx Error when Performing Schema Migrations
-
-You might get this error message during migration,
-
-```
-Encountered unknown type for Mysql: enum
-thread 'main' panicked at 'internal error: entered unreachable code: Mysql only supports a closed set of types.
-                         If you ever see this error message please open an issue at https://github.com/diesel-rs/diesel 
-                         containing a dump of your schema definition.', diesel_cli/src/print_schema.rs:310:17
-note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
-```
-
-I haven't found the cause (it doesn't happen on newer installs), but the migrations appear to work regardless 
-of this error message. You can essentially ignore it.
-
-### [Fix] Can't connect to local MySQL after install on Ubuntu
-
-- If MySql in local dev can't be connected to, [reset the accounts](https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux).

_develop/docs/local_development/local_development_k8s.md

@@ -1,22 +0,0 @@
-# Local Kubernetes environment for standing up the APIs and their dependencies
-
-(These are Paul's notes.)
-
-- Install minikube, kubectl, docker, and musl from your OS package manager
-- `rustup target add x86_64-unknown-linux-musl`
-- `cargo build --release --bin storyteller-web --target=x86_64-unknown-linux-musl`
-- `minikube start --extra-config=apiserver.service-node-port-range=1-65535`
-- `eval $(minikube docker-env)`
-- `minikube mount /path/to/storyteller-rust/:/storyteller-rust &> /dev/null &`
-- `docker build -t storyteller-web -f localdev/Dockerfile .`
-- `kubectl apply -f localdev/kubernetes.yml`
-- Get the ip address of minikube with `minikube ip`, this is where all the services will be running on their 
-regular ports (12345 for the api, 6379 for redis, 3306 for mysql)
-- That's it, we just spun up mysql, applied database migrations, spun up redis, and spun up our api
-
-# Development tips
-- Rebuild and "deploy" (locally) with `cargo build --release --bin storyteller-web --target=x86_64-unknown-linux-musl && kubectl rollout restart storyteller-web`
-- View logs from the api with `kubectl logs  storyteller-web<TAB>`
-- Delete everything minikube (if you start running out of disk space or what-have-you) with `minikube delete`
-- See pod status with `kubectl get pods`
-- If there are errors you can get more info with `kubectl describe pods`

_develop/docs/local_development/local_development_nginx_proxy.md

@@ -1,56 +0,0 @@
-Local Development nginx proxy
-=============================
-
-It isn't necessary to use nginx for development, but it can be nice. Nginx allows for a more 
-sophisticated testing setup that makes cookie and CORS issues less of a pain.
-
-You can use nginx to serve (locally-signed) SSL, redirect traffic to different ports, or run 
-multiple copies of the app side by side.
-
-### Hosts file
-
-Set the following configs in your `/etc/hosts` file (as you would probably do even without nginx),
-
-```
-127.0.0.1    dev.fakeyou.com
-127.0.0.1    api.dev.fakeyou.com
-127.0.0.1    devproxy.fakeyou.com
-
-127.0.0.1    dev.storyteller.ai
-127.0.0.1    api.dev.storyteller.ai
-127.0.0.1    devproxy.storyteller.ai
-
-# Deprecated development hostnames :
-127.0.0.1    jungle.horse
-127.0.0.1    api.jungle.horse
-```
-
-### Nginx installation and administration
-
-Configure Nginx per the checked in Nginx configs (and instructions) in
-the directory `/_develop/localdev/nginx-http-config`.
-
-This should be straightforward, but if it isn't, contribute docs here.
-
-### [Fix] Python 3.6 on Apple M1 Mac
-
-Python3.6 isn't supported on Apple silicon, and it's not in homebrew. It can be installed with 
-[nix using Rosetta](https://stackoverflow.com/a/65980989):
-
-Download: https://nixos.org/download.html#nix-quick-install
-
-```
-nix run nixpkgs.python36 -c python
-```
-
-Install venv:
-
-```
-nix run nixpkgs.python36 -c python -m venv python
-```
-
-Install other packages on Mac that aren't used in venv:
-
-```
-python3 -m pip install --user requests gdown youtube_dl
-```

_develop/docs/production/production_database.md

@@ -1,17 +0,0 @@
-Production Database
-===================
-
-Secrets can be subbed in for `.env` and the standard diesel migration tooling will work.
-
-Dump a MySQL report
--------------------
-
-```bash
-mysql -u storyteller \
-  -pPasswordHere \
-  -h IP_HERE \
-  -P 3306 \
-  -D storyteller \
-  -B \
-  -e "select * from voice_clone_requests;" | sed "s/'/\'/;s/\t/\",\"/g;s/^/\"/;s/$/\"/;s/\n//g" > voice_clone_report.csv
-```

_develop/localdev/Dockerfile

@@ -1,23 +0,0 @@
-FROM rust:latest
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && \
-    apt-get install -y \
-        mariadb-client \
-        imagemagick  \
-        libmariadb-dev \
-    && rm -rf /var/lib/apt/lists/*
-
-
-RUN cargo install diesel_cli --features mysql && \
-  cargo install sqlx-cli --features mysql
-
-WORKDIR /storyteller-rust
-
-ARG GIT_SHA
-
-RUN echo -n ${GIT_SHA} > /GIT_SHA
-
-CMD ["/storyteller-rust/localdev/startup.sh"]
-

_develop/localdev/kubernetes.yml

@@ -1,153 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata: 
-  name: mysql
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: mysql
-  template:
-    metadata:
-      labels:
-        app: mysql
-    spec:
-      containers:
-      - name: mysql
-        image: mysql:8.0.30
-        env:
-        - name: MYSQL_USER
-          value: storyteller
-        - name: MYSQL_PASSWORD
-          value: password
-        - name: MYSQL_DATABASE
-          value: storyteller
-        - name: MYSQL_ROOT_PASSWORD
-          value: password
-        ports:
-          - containerPort: 3306 
-
----
-
-apiVersion: apps/v1
-kind: Deployment
-metadata: 
-  name: redis
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: redis
-  template:
-    metadata:
-      labels:
-        app: redis
-    spec:
-      containers:
-      - name: redis
-        image: redis:7.0.4
-        ports:
-          - containerPort: 6379
-
----
-
-apiVersion: apps/v1
-kind: Deployment
-metadata: 
-  name: storyteller-web
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: storyteller-web
-  template:
-    metadata:
-      labels:
-        app: storyteller-web
-    spec:
-      containers:
-      - name: storyteller-web
-        image: storyteller-web:latest
-        imagePullPolicy: Never
-        env:
-          - name: MYSQL_URL
-            value: mysql://storyteller:password@mysql.default/storyteller
-          - name: REDIS_0_URL
-            value: redis://redisd@redis.default/0
-          - name: REDIS_1_URL
-            value: redis://redisd@redis.default/1
-          - name: ACCESS_KEY
-            value: garbage
-          - name: SECRET_KEY
-            value: garbage
-          - name: REGION_NAME
-            value: garbage
-          - name: W2L_PRIVATE_DOWNLOAD_BUCKET_NAME
-            value: garbage
-          - name: W2L_PUBLIC_DOWNLOAD_BUCKET_NAME
-            value: garbage
-          - name: AUDIO_UPLOADS_BUCKET_ROOT
-            value: garbage
-          - name: TWITCH_USERNAME
-            value: garbage
-          - name: TWITCH_STREAM_KEY
-            value: garbage
-          - name: TWITCH_APP_CLIENT_ID
-            value: garbage
-          - name: TWITCH_APP_CLIENT_SECRET
-            value: garbage
-        ports:
-          - containerPort: 12345
-        volumeMounts:
-          - mountPath: /storyteller-rust
-            name: storyteller-rust-folder
-      volumes:
-        - name: storyteller-rust-folder
-          hostPath:
-            path: /storyteller-rust
----
-
-apiVersion: v1
-kind: Service
-metadata: 
-  name: mysql
-spec:
-  type: NodePort
-  ports:
-    - name: mysql
-      port: 3306
-      nodePort: 3306
-  selector:
-    app: mysql
-
----
-
-apiVersion: v1
-kind: Service
-metadata: 
-  name: redis
-spec:
-  type: NodePort
-  ports:
-    - name: redis
-      port: 6379
-      nodePort: 6379
-  selector:
-    app: redis
-
----
-
-apiVersion: v1
-kind: Service
-metadata: 
-  name: storyteller-web
-spec:
-  type: NodePort
-  ports:
-    - name: storyteller-web
-      port: 12345
-      nodePort: 12345
-  selector:
-    app: storyteller-web
-
-

_develop/localdev/nginx-http-config/storyteller-ssl

@@ -1,258 +0,0 @@
-# Nginx configs for "jungle.horse" and "api.jungle.horse" (http)
-#
-# On Ubuntu Linux, this config is located at the following location:
-# 
-#  - /etc/nginx/sites-enabled/{name-of-site}
-#
-# Move this file there, and then restart with,
-#
-#   sudo service nginx restart
-#
-# On Mac, this is located at the following locations:
-#
-#  - /opt/homebrew/etc/nginx/servers/{name-of-site} (for nginx installed with homebrew)
-#  - /usr/local/etc/nginx/sites-enabled/{name-of-site} (for default nginx install)
-#
-#
-# On Mac, verify configs and restart with,
-#
-#   sudo nginx -t
-#   sudo nginx -s reload
-#
-#           ================
-#           SSL Instructions
-#           ================
-#
-# You can launch Chrome in a way it disregards self-signed certs and CORS:
-#
-#     chromium  --disable-web-security --ignore-certificate-errors --user-data-dir=~/chromeTemp
-#
-# To create self-signed certs on Ubuntu Linux,
-#
-#     sudo apt-get install openssl
-#
-#     sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/localhost.key -out /etc/ssl/certs/localhost.crt
-#
-#     ^ most fields can be blank, but the "Common Name" should be `*.jungle.horse`
-#
-#     sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/devfakeyou.key -out /etc/ssl/certs/devfakeyou.crt
-#     sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/devstoryteller.key -out /etc/ssl/certs/devstoryteller.crt
-#
-#     ^ most fields can be blank, but the "Common Name" should be `*.dev.fakeyou.com` and `*.dev.storyteller.io` respectively.
-#
-# For more: https://www.cloudsavvyit.com/1306/how-to-create-and-use-self-signed-ssl-on-nginx/
-#
-# To create self-signed certs on Mac,
-#
-#     openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt
-#
-#     ^ most fields can be blank, but the "Common Name" should be `*.jungle.horse`
-#
-# On Mac, load the self signed cert into the keychain with the following command:
-#
-#     sudo security add-trusted-cert \
-#      -d -r trustRoot \
-#      -k /Library/Keychains/System.keychain /etc/ssl/certs/localhost.crt
-#
-# Self signed certs might be considered insecure by the browser. In Firefox, 
-# set about:config `security.tls.insecure_fallback_hosts` to include a list of
-# permitted domains. You may also need to configure other settings. 
-#
-#
-
-# ========================================================
-#                      FAKEYOU
-# ========================================================
-
-# FakeYou javascript frontend over plain HTTP (launched by Yarn)
-server {
-    listen 80;
-    server_name dev.fakeyou.com;
-    location / {
-        proxy_set_header Host $host;
-        proxy_pass http://127.0.0.1:7000;
-        proxy_redirect off;
-    }
-}
-
-# FakeYou API backend over plain HTTP (Rust app)
-server {
-    listen 80;
-    server_name api.dev.fakeyou.com;
-    location / {
-        proxy_set_header Host $host;
-        proxy_pass http://127.0.0.1:12345;
-        proxy_redirect off;
-    }
-}
-
-# FakeYou javascript frontend over SSL (launched by Yarn)
-server {
-    listen 443 ssl;
-    listen [::]:443 ssl;
-    ssl_certificate /etc/ssl/certs/devfakeyou.crt;
-    ssl_certificate_key /etc/ssl/private/devfakeyou.key;
-    ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
-    server_name dev.fakeyou.com;
-    location / {
-        proxy_set_header Host $host;
-        proxy_pass http://127.0.0.1:7000;
-        proxy_redirect off;
-    }
-}
-
-# FakeYou API backend over SSL (Rust app)
-server {
-    listen 443 ssl;
-    listen [::]:443 ssl;
-    ssl_certificate /etc/ssl/certs/devfakeyou.crt;
-    ssl_certificate_key /etc/ssl/private/devfakeyou.key;
-    ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
-    server_name api.dev.fakeyou.com;
-    location / {
-        proxy_set_header Host $host;
-        proxy_pass http://127.0.0.1:12345;
-        proxy_redirect off;
-    }
-}
-
-# ========================================================
-#                      STORYTELLER
-# ========================================================
-
-# Storyteller javascript frontend over plain HTTP (launched by Yarn)
-server {
-    listen 80;
-    server_name dev.storyteller.io;
-    location / {
-        proxy_set_header Host $host;
-        proxy_pass http://127.0.0.1:7000;
-        proxy_redirect off;
-    }
-}
-
-# Storyteller API backend over plain HTTP
-server {
-    listen 80;
-    server_name api.dev.storyteller.io;
-    location / {
-        proxy_set_header Host $host;
-        proxy_pass http://127.0.0.1:12345;
-        proxy_redirect off;
-    }
-}
-
-# Storyteller javascript frontend over SSL (launched by Yarn)
-server {
-    listen 443 ssl;
-    listen [::]:443 ssl;
-    ssl_certificate /etc/ssl/certs/devstoryteller.crt;
-    ssl_certificate_key /etc/ssl/private/devstoryteller.key;
-    ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
-    server_name dev.storyteller.io;
-    location / {
-        proxy_set_header Host $host;
-        proxy_pass http://127.0.0.1:7000;
-        proxy_redirect off;
-    }
-}
-
-# Storyteller API backend over SSL
-server {
-    listen 443 ssl;
-    listen [::]:443 ssl;
-    ssl_certificate /etc/ssl/certs/devstoryteller.crt;
-    ssl_certificate_key /etc/ssl/private/devstoryteller.key;
-    ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
-    server_name api.dev.storyteller.io;
-    location / {
-        proxy_set_header Host $host;
-        proxy_pass http://127.0.0.1:12345;
-        proxy_redirect off;
-    }
-}
-
-# ========================================================
-#            OLD PRODUCTS THAT NEED UPDATING
-# ========================================================
-
-#     ## TODO(2022-02-13): This is new and untested
-#     # WebSocket API backend over plain HTTP
-#     server {
-#         listen 80;
-#         server_name ws.jungle.horse;
-#         location / {
-#             proxy_set_header Host $host;
-#             proxy_pass http://127.0.0.1:54321;
-#             proxy_redirect off;
-#         }
-#     }
-#
-#     # Websocket API backend over SSL
-#     server {
-#         listen 443 ssl;
-#         listen [::]:443 ssl;
-#         ssl_certificate /etc/ssl/certs/localhost.crt;
-#         ssl_certificate_key /etc/ssl/private/localhost.key;
-#         ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
-#         server_name ws.jungle.horse;
-#         location / {
-#             proxy_set_header Host $host;
-#             proxy_pass http://127.0.0.1:54321;
-#             proxy_redirect off;
-#         }
-#     }
-#
-#     # Javascript frontend over plain HTTP
-#     server {
-#         listen 80;
-#         server_name jungle.horse;
-#         location / {
-#             proxy_set_header Host $host;
-#             proxy_pass http://127.0.0.1:7000;
-#             proxy_redirect off;
-#         }
-#     }
-#
-#     # TODO(2022-02-13): This is new and untested
-#     # Javascript frontend (OBS Layer) over plain HTTP
-#     server {
-#         listen 80;
-#         server_name obs.jungle.horse;
-#         location / {
-#             proxy_set_header Host $host;
-#             proxy_pass http://127.0.0.1:7007;
-#             proxy_redirect off;
-#         }
-#     }
-#
-#     # Javascript frontend over SSL
-#     server {
-#         listen 443 ssl;
-#         listen [::]:443 ssl;
-#         ssl_certificate /etc/ssl/certs/localhost.crt;
-#         ssl_certificate_key /etc/ssl/private/localhost.key;
-#         ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
-#         server_name jungle.horse;
-#         location / {
-#             proxy_set_header Host $host;
-#             proxy_pass http://127.0.0.1:7000;
-#             proxy_redirect off;
-#         }
-#     }
-#
-#     # Javascript frontend over SSL (OBS Layer)
-#     server {
-#         listen 443 ssl;
-#         listen [::]:443 ssl;
-#         ssl_certificate /etc/ssl/certs/localhost.crt;
-#         ssl_certificate_key /etc/ssl/private/localhost.key;
-#         ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
-#         server_name obs.jungle.horse;
-#         location / {
-#             proxy_set_header Host $host;
-#             proxy_pass http://127.0.0.1:7007;
-#             proxy_redirect off;
-#         }
-#     }
-#

_develop/localdev/startup.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-set +e +o pipefail
-diesel migration run --database-url="${MYSQL_URL}"
-set -e -o pipefail
-./target/x86_64-unknown-linux-musl/release/storyteller-web
-

_develop/localdev/ubuntu_setup.sh

@@ -1,36 +0,0 @@
-#!/bin/bash
-
-echo 'Installing APT packages...'
-
-# OpenSSL requirements: libssl-dev, pkg-config
-# Database requirements: mysql-server, libmysqlclient-dev
-# Other tools: jq (db codegen)
-sudo apt-get install -y \
-  jq \
-  libmysqlclient-dev \
-  libssl-dev \
-  mysql-server \
-  pkg-config \
-  --no-install-recommends
-
-echo 'Installing Cargo packages...'
-
-# Install Diesel CLI for migrations
-cargo install diesel_cli \
-  --no-default-features \
-  --features mysql
-
-# Install SQLx for codegen
-cargo install sqlx-cli \
-  --no-default-features \
-  --features rustls,mysql
-
-echo 'Installing Database user...'
-
-# Create DB account (using default system mysql password)
-sudo mysql \
-  -u "root" \
-  -p"password" \
-  -e "use mysql; CREATE DATABASE IF NOT EXISTS storyteller; CREATE USER IF NOT EXISTS 'storyteller'@'localhost' IDENTIFIED BY 'password'; GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';"
-
-

_develop/tools/.gitignore

@@ -1,6 +0,0 @@
-# Secrets
-db_secrets.toml
-
-# Python Virtual Env
-python/
-

_develop/tools/find_101soundboards_accounts.py

@@ -1,100 +0,0 @@
-#!/usr/bin/env python
-
-# FIXME: Needs packages 'toml' and 'mysql-connector-python'. Put in requirements.txt.
-
-import time
-import toml
-import mysql.connector
-
-DB_SECRETS_FILE = 'db_secrets.toml'
-
-# TODO: Share db-related setup when more scripts are added.
-
-def read_database_secrets(secrets_file = DB_SECRETS_FILE):
-    secrets = None
-    with open(secrets_file, "r") as f:
-        secrets = toml.load(f)
-
-    required_keys = ['database', 'hostname', 'port', 'password', 'username']
-    for key in required_keys:
-        if key not in secrets:
-            raise Exception(f"Required key '#{key}' not present")
-
-    return secrets
-
-def connect_to_mysql():
-    secrets = read_database_secrets()
-    connection = mysql.connector.connect(
-        host = secrets['hostname'],
-        port = secrets['port'],
-        user = secrets['username'],
-        password = secrets['password'],
-        database = secrets['database'],
-    )
-    return connection
-
-connection = connect_to_mysql()
-cursor = connection.cursor()
-
-SAFE_ID = 76000000 # Reasonably completed on 2022-06-24
-LIMIT = 100000
-
-#def get_record_count():
-#    query = f"SELECT COUNT(*) FROM tts_inference_jobs"
-#    cursor.execute(query)
-#    result = cursor.fetchall()
-#    print(result)
-#    return result[0]
-#
-##total_count = get_record_count()
-
-#total_count = 75082057
-total_count = 21925602
-
-def get_user_tokens(cursor):
-    query = f"""
-		SELECT
-			r.maybe_creator_user_token, r.creator_ip_address
-		FROM
-			tts_results as r
-				LEFT OUTER JOIN
-			users as u
-			ON
-					r.maybe_creator_user_token = u.token
-		WHERE
-				r.raw_inference_text LIKE '%spidermant%'
-		AND r.created_at > ( CURDATE() - INTERVAL 1 HOUR )
-    """
-    cursor.execute(query)
-    results = cursor.fetchall()
-
-    user_tokens = [row[0] for row in results]
-    user_tokens = [token for token in user_tokens if token is not None]
-    user_tokens = set(user_tokens)
-
-    ip_addresses = [row[1] for row in results]
-    ip_addresses = [ip for ip in ip_addresses if ip is not None]
-    ip_addresses = set(ip_addresses)
-
-    return (user_tokens, ip_addresses)
-
-(user_tokens, ip_addresses) = get_user_tokens(cursor)
-
-print(user_tokens)
-print(len(user_tokens))
-print(len(ip_addresses))
-
-with open('101_user_tokens.txt', 'w') as f:
-    for user_token in user_tokens:
-        f.write(user_token)
-        f.write("\n")
-
-with open('101_ip_addresses.txt', 'w') as f:
-    for ip in ip_addresses:
-        f.write(ip)
-        f.write("\n")
-
-
-
-
-

_develop/tools/requirements.txt

@@ -1,3 +0,0 @@
-mysql-connector-python==8.0.32
-protobuf==3.20.3
-toml==0.10.2

_docs/database_schema.md

@@ -1,8 +1,6 @@
 database schema
 ================
 
-(This is woefully incomplete and unmaintained.)
-
 Twitch Model
 ------------
 

_docs/local_development_setup.md

@@ -0,0 +1,207 @@
+dev setup
+=========
+
+[Back to main README](../README.md)
+
+### Debugging and Fixing CUDA/PyTorch
+
+See this spreadsheet for previous battles with version incompatibility:
+https://docs.google.com/spreadsheets/d/1BEdLmwOzo3r83-iJn9sj6co1VT92t3lIeJ-lNH25bdQ/edit#gid=0
+
+Ubuntu might upgrade the driver by accident, and it might need reinstallation. Who knows.
+This stuff is a nightmare.
+
+### Database Setup
+
+Install the following libraries, and see the notes further below about MySQL on Ubuntu 20.04.
+
+```
+mysql-server
+imagemagick
+```
+
+To manage the database and perform migrations, install the Rust tools diesel and sqlx.
+
+We'll be using diesel to manage the migrations, but sqlx within the app to actually perform queries.
+Diesel is an ORM, which is dumb, so we use sqlx as at-compile-time typesafe SQL.
+
+```
+sudo apt-get install libmysqlclient-dev
+cargo install sqlx-cli --no-default-features --features mysql [2022-01-16: is this needed?]
+```
+
+To install a more modern version of Diesel (unreleased) that 
+supports the "migrations_dir" toml config option:
+
+```
+cargo install diesel_cli \
+  --git https://github.com/diesel-rs/diesel.git \
+  --rev a213fe232a122f35a812b0ce0269708a1845a4c9 \
+  --no-default-features \
+  --features mysql
+```
+
+#### Sqlx Error on Linux when Performing Schema Migrations
+
+You might get this error message during migration,
+
+```
+Encountered unknown type for Mysql: enum
+thread 'main' panicked at 'internal error: entered unreachable code: Mysql only supports a closed set of types.
+                         If you ever see this error message please open an issue at https://github.com/diesel-rs/diesel containing a dump of your schema definition.', diesel_cli/src/print_schema.rs:310:17
+note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
+```
+
+I haven't found the cause (it doesn't happen on Mac), but the migrations appear to work regardless of the error message.
+
+#### Linux database notes
+
+If MySql in local dev can't be connected to, reset the accounts:
+
+https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux
+
+#### Fixing dev MySql on Ubuntu 20.04
+
+For some reason, the MySql default install on 20.04 gave me a bunch of trouble.
+
+In retrospect, I _think_ this is because 'root@localhost' requires sudo to access, but if this
+gives any trouble in the future, here's how I got around it (two full hours of distraction!)
+
+```
+sudo apt-get install mysql-server
+
+# But for some reason the default password doesn't work and diesel/sqlx can't connect?
+
+# Kill everything
+
+sudo apt purge mysql-server mysql-client mysql-common
+sudo apt autoremove
+sudo mv -iv /var/lib/mysql /var/tmp/mysql-backup
+sudo rm -rf /var/lib/mysql*
+
+sudo /usr/bin/mysql_secure_installation
+
+# still no work...
+
+sudo systemctl stop mysql.service
+
+# ugh, it wasn't chowned or created (both states observed in different installs)…
+sudo mkdir -p /var/run/mysqld
+sudo chown mysql:mysql /var/run/mysqld
+
+sudo mysqld_safe --skip-grant-tables --skip-networking &
+
+mysql -u root
+
+flush privileges;
+USE mysql;
+ALTER USER 'root'@'localhost' IDENTIFIED BY 'root';
+
+sudo killall -u mysql
+sudo systemctl restart mysql.service
+
+# now it works
+sudo mysql -u root -p
+
+# but now I have to use "sudo" !?!?
+
+use mysql;
+
+CREATE USER 'storyteller'@'localhost' IDENTIFIED BY 'password';
+GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';
+```
+
+### Install Diesel (migrations only)
+
+We use Diesel to manage migrations, but we don't use it in server code.
+Actual server code uses SQLx. To install the CLI tool, run the following:
+
+```
+cargo install diesel_cli
+```
+
+Mac [has issues](https://github.com/diesel-rs/diesel/issues/2605) and requires a few dependencies:
+
+```
+brew install libpq
+```
+
+### Database migrations
+
+To reset the entire database (drop, migrate), run:
+
+```
+diesel database reset
+```
+
+To migrate at the current step and beyond:
+
+```
+diesel migration run
+```
+
+To undo migrations at the current step:
+
+```
+diesel migration redo
+```
+
+### Server Query Codegen
+
+We use SQLx instead of Diesel in the production server. It's typesafe
+SQL instead of an ORM like Diesel.
+
+SQLx connects to a database to derive type information, but obviously
+cannot do this for builds in CI. In order to cache the types, we build
+and check in a cache file (necessary for builds):
+
+```
+SQLX_OFFLINE=true cargo sqlx prepare
+```
+
+Now that we have multiple binaries, it's required to include all the queries in the main
+binary so we can generate the cached queries as a single target. That's then executed
+with:
+
+```
+SQLX_OFFLINE=true cargo sqlx prepare -- --bin storyteller-web
+```
+
+### Setting Up a Local Nginx Proxy
+
+Set up a local nginx to proxy to the frontend and backend so cookie issues aren't annoying
+
+Configure Nginx per the checked in Nginx configs (and instructions) in `localdev/nginx-http-config`.
+
+```
+
+And in /etc/hosts,
+
+```
+127.0.0.1  jungle.horse
+127.0.0.1  api.jungle.horse
+
+```
+
+### Python 3.6 on Apple M1 Mac
+
+Python3.6 isn't supported on Apple silicon, and it's not in homebrew. It can be installed with 
+[nix using Rosetta](https://stackoverflow.com/a/65980989):
+
+Download: https://nixos.org/download.html#nix-quick-install
+
+```
+nix run nixpkgs.python36 -c python
+```
+
+Install venv:
+
+```
+nix run nixpkgs.python36 -c python -m venv python
+```
+
+Install other packages on Mac that aren't used in venv:
+
+```
+python3 -m pip install --user requests gdown youtube_dl
+```

_migrations/2021-05-15-071534_users/up.sql

@@ -0,0 +1,235 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+CREATE TABLE users (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Visible "primary key"
+  token VARCHAR(32) NOT NULL,
+
+  -- Username is a lookup key; display_name allows the user to add custom case.
+  username VARCHAR(20) NOT NULL,
+  display_name VARCHAR(20) NOT NULL,
+
+  email_address VARCHAR(255) NOT NULL,
+  email_confirmed BOOLEAN NOT NULL DEFAULT false,
+
+  -- The role assigned to the user confers permissions.
+  user_role_slug VARCHAR(16) NOT NULL,
+
+  -- ========== CREDENTIALS ==========
+
+  -- Bcrypt password hash. Granted, there are newer methods:
+  -- https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html
+  -- Passwords may be a max of 64 characters per BCrypt.
+  password_hash BINARY(60) NOT NULL,
+
+  -- Incremented with every update to the password.
+  password_version INT NOT NULL DEFAULT 0,
+
+  -- ========== ABUSE TRACKING ==========
+
+  -- For abuse tracking.
+  -- Wide enough for IPv4/6
+  ip_address_creation VARCHAR(40) NOT NULL,
+  ip_address_last_login VARCHAR(40) NOT NULL,
+  ip_address_last_update VARCHAR(40) NOT NULL,
+
+  -- ========== DISPLAY / PROFILE ==========
+
+  -- Gravatar image hashes are precomputed.
+  email_gravatar_hash CHAR(32) NOT NULL,
+
+  -- The profile of the user in markdown (user editable).
+  profile_markdown TEXT NOT NULL,
+
+  -- Generated HTML (not user-editable).
+  profile_rendered_html TEXT NOT NULL,
+
+  -- An uploaded avatar. Public hash in our bucket.
+  -- avatar_public_bucket_hash CHAR(32) DEFAULT NULL,
+
+  -- ========== SOCIAL MEDIA ==========
+
+  -- Social media usernames
+  -- These are not confirmed. We'll need to build an OAuth system to handle that.
+  discord_username VARCHAR(36) DEFAULT NULL,
+  twitter_username VARCHAR(36) DEFAULT NULL,
+  twitch_username VARCHAR(36) DEFAULT NULL,
+  patreon_username VARCHAR(36) DEFAULT NULL,
+  github_username VARCHAR(36) DEFAULT NULL,
+  cashapp_username VARCHAR(36) DEFAULT NULL,
+  website_url VARCHAR(255) DEFAULT NULL,
+
+  -- ========== USER PREFERENCES ==========
+
+  -- If the user doesn't want to use gravatar and doesn't have an uploaded avatar.
+  disable_gravatar BOOLEAN NOT NULL DEFAULT false,
+
+  -- (THIS MIGHT NOT BE USED)
+  -- NB: DO NOT SORT!
+  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
+  -- Visibility preference used at TTS inference time (which can be changed on the result itself).
+  -- Moderators will still see them.
+  preferred_tts_result_visibility ENUM(
+    'public',
+    'hidden',
+    'private'
+  ) NOT NULL DEFAULT 'public',
+
+  -- (THIS MIGHT NOT BE USED)
+  -- NB: DO NOT SORT!
+  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
+  -- Visibility preference used at TTS inference time (which can be changed on the result itself).
+  -- Moderators will still see them.
+  preferred_w2l_result_visibility ENUM(
+    'public',
+    'hidden',
+    'private'
+  ) NOT NULL DEFAULT 'public',
+
+  -- Auto play preferences
+  -- Tri-state because we want to potentially change the default behavior.
+  auto_play_audio_preference BOOLEAN DEFAULT NULL,
+  auto_play_video_preference BOOLEAN DEFAULT NULL,
+
+  -- Favorite TTS voice to use by default
+  -- maybe_preferred_tts_model_token VARCHAR(32) DEFAULT NULL,
+
+  -- Favorite W2L model to use by default
+  -- maybe_preferred_w2l_template_token VARCHAR(32) DEFAULT NULL,
+
+  -- Settings
+  -- DO NOT REORDER.
+  -- dark_mode_preference ENUM(
+  --   'light-mode',
+  --   'dark-mode',
+  --   'use-clock'
+  --  ) NOT NULL DEFAULT 'light-mode',
+
+  -- ========== STATS ==========
+
+  -- For tracking stats.
+  -- The "cached" values are updated by a background job.
+  -- cached_tts_rendered_counter INT(10) NOT NULL DEFAULT 0,
+  -- cached_w2l_rendered_counter INT(10) NOT NULL DEFAULT 0,
+
+  -- ========== MODERATION DETAILS ==========
+
+  -- Different than deleted.
+  -- Users still show up, but can't do anything.
+  is_banned BOOLEAN NOT NULL DEFAULT false,
+
+  -- If a moderator has comments.
+  maybe_mod_comments VARCHAR(255) DEFAULT NULL,
+
+  -- The last moderator that made changes.
+  maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
+
+  -- ========== VECTOR CLOCK ==========
+
+  -- Incremented with every update.
+  version INT NOT NULL DEFAULT 0,
+
+  -- ========== TIMESTAMPS ==========
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- If the user is deleted, we set this.
+  -- This is different than banned. These users won't show up at all.
+  user_deleted_at TIMESTAMP NULL,
+  mod_deleted_at TIMESTAMP NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (token),
+  UNIQUE KEY (username),
+  UNIQUE KEY (email_address),
+  KEY fk_user_role_slug (user_role_slug)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
+
+CREATE TABLE user_roles (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Effective "primary key"
+  slug CHAR(16) NOT NULL,
+
+  name VARCHAR(255) NOT NULL,
+
+  -- Usage
+  can_use_tts BOOLEAN NOT NULL DEFAULT FALSE,
+  can_use_w2l BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_own_tts_results BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_own_w2l_results BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_own_account BOOLEAN NOT NULL DEFAULT FALSE,
+
+  -- Contribution
+  can_upload_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
+  can_upload_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_own_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_own_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
+
+  -- Moderation
+  can_approve_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
+  -- Edit includes "lock_from_use" and "lock_from_modification"
+  can_edit_other_users_profiles BOOLEAN NOT NULL DEFAULT FALSE,
+  can_edit_other_users_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
+  can_edit_other_users_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_other_users_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_other_users_tts_results BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_other_users_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_other_users_w2l_results BOOLEAN NOT NULL DEFAULT FALSE,
+  can_ban_users BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_users BOOLEAN NOT NULL DEFAULT FALSE,
+
+  -- Incremented with every update.
+  version INT NOT NULL DEFAULT 0,
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (slug)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
+
+CREATE TABLE user_sessions (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Session entropy
+  token VARCHAR(32) NOT NULL,
+
+  -- Foreign key to user
+  user_token VARCHAR(32) NOT NULL,
+
+  -- Track each session's creation IP.
+  -- Wide enough for IPv4/6
+  ip_address_creation VARCHAR(40) NOT NULL,
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- Session termination time.
+  -- This must be set by the server code, or the session is invalid
+  expires_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+  -- deletion = session termination
+  -- Typically these are destroyed by users, but if in the future we allow mods to
+  -- delete them, it doesn't really matter who did the deletion: sessions are not
+  -- designed to be recoverable.
+  deleted_at TIMESTAMP NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (token),
+  KEY fk_user_token (user_token),
+  KEY index_ip_address_creation (ip_address_creation)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_migrations/2022-08-05-202017_generic_download_jobs/up.sql

@@ -0,0 +1,110 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+CREATE TABLE generic_download_jobs (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Effective "primary key" (PUBLIC)
+  -- This is so the in-progress results can be looked up by the UI.
+  token VARCHAR(32) NOT NULL,
+
+  -- Idempotency token
+  -- This is so the frontend client doesn't submit duplicate jobs.
+  uuid_idempotency_token VARCHAR(36) NOT NULL,
+
+  -- ========== SUCCESS CASE ==========
+
+  -- If the job completes successfully, this is the result token.
+  -- This is only populated on a successful result.
+  on_success_downloaded_entity_token VARCHAR(32) DEFAULT NULL,
+
+  -- The type of the object will vary based on the type of the upload,
+  -- and we may include heuristics that auto-detect types in the future
+  on_success_downloaded_entity_type VARCHAR(32) DEFAULT NULL,
+
+  -- ========== UPLOAD DETAILS ==========
+
+  -- A "well defined" type of upload
+  -- The first upload type will be "hifigan_model"
+  download_type VARCHAR(32) NOT NULL,
+
+  -- Assume the same URL can be uploaded more than once (in case of errors or
+  -- people sharing models)
+
+  download_url VARCHAR(512) NOT NULL,
+
+  -- The user-supplied title of the upload
+  -- This might have different meanings for different upload types
+  -- The "name" of the voice model, which might be complicated.
+  -- If maybe_subject_token (etc.) is set, then it's authoritative instead.
+  title VARCHAR(255) NOT NULL,
+
+  -- ========== CREATOR DETAILS AND PREFERENCES ==========
+
+  -- Foreign key to user
+  creator_user_token VARCHAR(32) NOT NULL,
+
+  -- For abuse tracking.
+  -- Wide enough for IPv4/6
+  creator_ip_address VARCHAR(40) NOT NULL,
+
+  -- The creator can set a desired visibility for their data.
+  -- This does not always apply to every upload type.
+  -- Additionally, some upload types may require moderator approval prior
+  -- to being publicly listed, and this field has no bearing on that.
+  -- NB: DO NOT CHANGE THE ORDER OF THE ENUM VALUES DURING SCHEMA MIGRATIONS.
+  creator_set_visibility ENUM(
+    'public',
+    'hidden',
+    'private'
+  ) NOT NULL DEFAULT 'public',
+
+  -- ========== JOB SYSTEM DETAILS ==========
+
+  -- Jobs begin as "pending", then transition to other states.
+  --
+  --  * Pending = job is ready to go
+  --  * Started = job is running
+  --  * Complete_Success = job is done (success)
+  --  * Complete_Failure = job is done (failure)
+  --  * Attempt_Failed = job failed but may retry.
+  --  * Dead = job failed permanently.
+  --
+  -- Pending -> Started -> Complete_Success
+  --                    |-> Complete_Failure
+  --                    \-> Attempt_Failed -> Started -> { Complete, Failed, Dead }
+  status ENUM(
+    'pending',
+    'started',
+    'complete_success',
+    'complete_failure',
+    'attempt_failed',
+    'dead') NOT NULL DEFAULT 'pending',
+
+  -- We can track this against a "max_attempt_count"
+  attempt_count INT(3) NOT NULL DEFAULT 0,
+
+  -- If there is a failure, tell the user why.
+  failure_reason VARCHAR(512) DEFAULT NULL,
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- Failed jobs will set a next attempt time.
+  -- Subsequent tries can increase the timeout.
+  -- Failures because of permissions require human intervention => [retry_at=null].
+  -- Failures because of invalid files are dead => [status=dead].
+  retry_at TIMESTAMP NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (token),
+  UNIQUE KEY (uuid_idempotency_token),
+  KEY fk_on_download_type (download_type),
+  KEY fk_creator_user_token (creator_user_token),
+  KEY index_status (status),
+  KEY index_creator_ip_address (creator_ip_address)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;