purge session cache with billing endpoints

Bt redis cache endpoint

.dockerignore

@@ -1,9 +1,11 @@
 *secret*sh
 *~
+.dockerignore
 .env
 .env-secrets
 .git
 .idea/
+Dockerfile
 __pycache__/
 _ignored_schema.rs
 models/Wav2Lip/python/

.env

@@ -1,7 +1,14 @@
+SERVER_ENVIRONMENT=development
+
 DATABASE_URL=mysql://storyteller:password@localhost/storyteller
 #DATABASE_URL=mysql://root:root@localhost/storyteller
 #DATABASE_URL=mysql://root@localhost/storyteller
 
+# Diesel-cli reads this for the migration directory to use.
+# We've renamed it so the migrations are sorted near the top of the project directory tree
+# (since they kind of serve as documentation)
+MIGRATION_DIRECTORY=_sql/migrations/
+
 # SQLX normally requires a live database to reflect upon at build time.
 # This setting turns that on and requires a `sql-data.json` cache file instead.
 # See https://github.com/launchbadge/sqlx/tree/master/sqlx-cli#readme
@@ -64,3 +71,14 @@ TTS_HIFIGAN_SUPERRES_VOCODER_MODEL_FILENAME=hifigan_superres_twilight_33000.pt
 
 # Development key. Permissible to check in.
 NEWRELIC_API_KEY=c402be1c4c4216ec83d0cfc0851520cfFFFFNRAL
+
+# Stripe configs
+# All secrets are development and okay to check in
+STRIPE_CHECKOUT_SUCCESS_URL_PATH=/checkout_success
+STRIPE_CHECKOUT_CANCEL_URL_PATH=/checkout_cancel
+STRIPE_PORTAL_RETURN_URL_PATH=/portal_success
+STRIPE_PORTAL_DEFAULT_CONFIG_ID=bpc_1LyPPREU5se17MekYiViZF12
+# STRIPE_PORTAL_DEFAULT_CONFIG_ID=bpc_1M4jc3EU5se17MekmgdS4dDJ # Production (not a secret)
+STRIPE_PUBLISHABLE_KEY=pk_test_51KZvWXEU5se17Mekx6e0gw32ovDF78k2eeaGctLIs8oQVxN6DerEp1BDqh6gLMYKli4VsTZoVGHoxq5RmwKMlxtP00r59UEEad
+STRIPE_SECRET_KEY=sk_test_51KZvWXEU5se17Mek11ZdnWr4frnyw7tWOPuvl3pRzL290zvkT6KqcbGUl3fcvc3hA8oH6lWXz2kGWXXUlNcCuDxB003uVT9JX0
+STRIPE_SECRET_WEBHOOK_SIGNING_KEY=whsec_40b60bd75768052532b63dbeefa7a1b163e182c3e657fb27fa906ef49a2cd59f

.env-download-job

@@ -0,0 +1,25 @@
+# Environmental configurations specific to `download-job`.
+# These are defaulted for dev and are easy to overwrite in production with K8s environment variables.
+# !! DOES *NOT* CONTAIN SECRETS !!
+
+WEB_DOWNLOADER_COMMAND=./download_internet_file.py
+WEB_DOWNLOADER_MAYBE_VENV_SCRIPT=./python/bin/activate
+WEB_DOWNLOADER_MAYBE_DOCKER_IMAGE=d73f28ce3ff6
+
+SOFTVC_MODEL_CHECK_ROOT_DIRECTORY=/
+SOFTVC_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate"
+SOFTVC_MODEL_CHECK_MAYBE_DOCKER_IMAGE=a79106991ce3
+
+TACOTRON_MODEL_CHECK_ROOT_DIRECTORY=/models/tts
+TACOTRON_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate"
+TACOTRON_MODEL_CHECK_COMMAND=./vocodes_model_check_tacotron.py
+TACOTRON_MODEL_CHECK_MAYBE_DOCKER_IMAGE=5642d0fd7fc1
+
+HIFIGAN_SOFTVC_MODEL_CHECK_ROOT_DIRECTORY=/
+HIFIGAN_SOFTVC_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate"
+HIFIGAN_SOFTVC_MODEL_CHECK_MAYBE_DOCKER_IMAGE=a79106991ce3
+
+HIFIGAN_MODEL_CHECK_ROOT_DIRECTORY=/models/tts
+HIFIGAN_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate"
+HIFIGAN_MODEL_CHECK_COMMAND=./vocodes_model_check_hifigan.py
+HIFIGAN_MODEL_CHECK_MAYBE_DOCKER_IMAGE=5642d0fd7fc1

.gitattributes

@@ -0,0 +1,5 @@
+# Ignore certain code for Github language statistics
+# https://github.com/github/linguist/blob/master/docs/overrides.md
+old_code/trumpet-trumped.com/dictionary/cmudict-0.7b linguist-vendored
+old_code/voder-vocodes-1.0/frontends/admin/node_modules linguist-vendored
+windows/sqlite linguist-vendored

.github/workflows/ci.yml

@@ -0,0 +1,52 @@
+name: CI
+on: [push, pull_request]
+env: 
+  MYSQL_URL: mysql://root:root@localhost/storyteller
+  DATABASE_URL: mysql://root:root@localhost/storyteller
+  REDIS_0_URL: redis://redisd@localhost/0
+  REDIS_1_URL: redis://redisd@localhost/1
+  DEBIAN_FRONTEND: noninteractive
+
+jobs:
+  setup_build_test:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+      - name: Cache cargo registry
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
+      - name: Setup mysql
+        run: |
+          sudo systemctl start mysql
+          sudo apt-get update
+          sudo apt-get -y install mysql-client libmysqlclient-dev
+          mysql -e "create user 'storyteller'@'localhost' IDENTIFIED BY 'password';" -uroot -proot 
+          mysql -e "create database storyteller; grant all on storyteller.* to 'storyteller'@'localhost';" -uroot -proot
+      - name: Setup Redis
+        run: |
+          sudo apt-get -y install redis-server
+          sudo service redis-server start
+      - name: Setup Rust
+        run: |
+          rustup set profile minimal
+          rustup show
+          rustup install 1.66
+          rustup default 1.66
+          rustup show
+          cargo install sqlx-cli --features openssl-vendored
+          cargo install diesel_cli
+      - name: Run database migrations
+        run: |
+          # nothing to see here ;)
+          set +e +o pipefail
+          diesel migration run
+          set -e -o pipefail
+      - name: Build 
+        run: cargo build --release
+      - name: Test
+        run: cargo test --release

.github/workflows/docker-publish.yml

@@ -15,53 +15,98 @@ env:
 jobs:
   # Push image to GitHub Packages.
   # See also https://docs.docker.com/docker-hub/builds/
-  push:
-    runs-on: ubuntu-latest
+  # See also https://depot.dev/blog/docker-layer-caching-in-github-actions
+  # See also https://evilmartians.com/chronicles/build-images-on-github-actions-with-docker-layer-caching
+  build-docker-image:
+    name: "Build Docker Image"
+    runs-on: ubuntu-22.04 # runs-on: ubuntu-latest
     if: github.event_name == 'push'
     permissions:
       packages: write
       contents: read
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
+      # - uses: docker/setup-buildx-action@v1
+      #   name: Set up Docker Buildx
+
+      - uses: docker/login-action@v1
+        name: Login to GitHub Container Registry
         with:
           registry: ghcr.io
-          #username: ${{ github.repository_owner }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }} 
+          username: ${{ github.actor }} # username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build image
+      - name: Calculate short SHA
+        id: vars
+        # echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
         run: |
-          SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
-          docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" --build-arg GIT_SHA=${SHORT_SHA}
+          echo "sha_short=$(echo ${{ github.sha }} | cut -c1-12)" >> $GITHUB_OUTPUT
 
-      - name: Push image to GitHub Container Registry
+      - name: Emit short SHA
         run: |
-          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+          echo "Full SHA: ${{ github.sha }}"
+          echo "Short SHA: ${{ steps.vars.outputs.sha_short }}"
 
-          # Change all uppercase to lowercase
-          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+      #- uses: actions/cache@v2
+      #  name: Cache Docker layers
+      #  with:
+      #    path: /tmp/.buildx-cache
+      #    key: ${{ runner.os }}-buildx-${{ github.sha }}
+      #    restore-keys: |
+      #      ${{ runner.os }}-buildx-
 
-          # Strip git ref prefix from version
-          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+      # Docs for this action are at https://github.com/marketplace/actions/build-docker-images-using-cache
+      - uses: whoan/docker-build-with-cache-action@v6
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }} # username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          image_name: storytold/storyteller-rust
+          # NB: The `git_push_tag: true` option does not work as expected, so we put the sha in `image_tag`:
+          # https://github.com/whoan/docker-build-with-cache-action/issues/72
+          image_tag: latest,${{ github.sha }},${{ steps.vars.outputs.sha_short }}
+          # NB: Inject the Git SHA into the container so the servers can emit their own version:
+          build_extra_args: "--build-arg=GIT_SHA=${{ github.sha }}"
 
-          # Strip "v" prefix from tag name
-          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+      # - name: Build image
+      #   run: |
+      #     SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
+      #     docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" --build-arg GIT_SHA=${SHORT_SHA}
 
-          # Use Docker `latest` tag convention
-          [ "$VERSION" == "$default-branch" ] && VERSION=latest
+      # - name: Push image to GitHub Container Registry
+      #   run: |
+      #     IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
 
-          echo IMAGE_ID=$IMAGE_ID
-          echo VERSION=$VERSION
+      #     # Change all uppercase to lowercase
+      #     IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
 
-          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
-          docker push $IMAGE_ID:$VERSION
+      #     # Strip git ref prefix from version
+      #     VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
 
-          SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
+      #     # Strip "v" prefix from tag name
+      #     [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
 
-          docker tag $IMAGE_NAME $IMAGE_ID:$SHORT_SHA
-          docker push $IMAGE_ID:$SHORT_SHA
+      #     # Use Docker `latest` tag convention
+      #     [ "$VERSION" == "$default-branch" ] && VERSION=latest
 
+      #     echo IMAGE_ID=$IMAGE_ID
+      #     echo VERSION=$VERSION
+
+      #     docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
+      #     docker push $IMAGE_ID:$VERSION
+
+      #     SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
+
+      #     docker tag $IMAGE_NAME $IMAGE_ID:$SHORT_SHA
+      #     docker push $IMAGE_ID:$SHORT_SHA
+
+      - name: Cleanup / Debug
+        # This job will run even on failures to help with debugging.
+        if: always()
+        run: |
+          echo "Report at final github action step:"
+          echo "Disk usage:"
+          pwd
+          du -hsc * | sort -hr

.gitignore

@@ -1,22 +1,51 @@
-*.swo
-*.swp
+# Secrets are BAD!
 *secret*sh
-*~
 .env
+.env-aichatbot-secrets
 .env-secrets
-.idea/
-__pycache__/
-_ignored_schema.rs
 db.secrets
 db_secrets.toml
-models/Wav2Lip/python/
-models/tacotron2/python/
 production_database_secret_connect.sh
 production_database_secret_query.sh
 production_database_voice_clone_report_download.sh
 secrets.toml
 secrets.yaml
-target/
-tools/.secrets
-tools/db_secrets.toml
+_develop/tools/.secrets
+_develop/tools/db_secrets.toml
 twitch_secrets.toml
+
+# Rust build directory
+target/
+crates/lib/database_queries/target/
+crates/lib/sqlite_queries/target/
+
+# Vim noise
+*.swo
+*.swp
+*~
+
+# IntelliJ/CLion noise
+.idea/
+
+# Python noise
+__pycache__/
+
+# We use diesel for DB migrations only. We don't care about its codgen.
+_ignored_schema.rs
+
+# AI Chat Sidecar has its own schema managed by diesel, but we don't care for diesel codegen.
+crates/programs/aichatbot-sidecar/_ignored_schema.rs
+
+# AI Chat sidecar's SQlite database shouldn't be checked in
+crates/programs/aichatbot-sidecar/database.db
+
+# Runtime data for the AI chatbot
+runtime_data/
+
+# Old Python virtual environments
+models/Wav2Lip/python/
+models/tacotron2/python/
+
+# Stuff I want to keep around, but not commit
+scraps/
+

.idea/dictionaries/bt.xml

@@ -1,18 +1,35 @@
 <component name="ProjectDictionaryState">
-  <!--
-  Despite spellcheck dictionaries being created per-user, IntelliJ loads all of them, per:
-  https://stackoverflow.com/a/28151692
-  -->
   <dictionary name="bt">
     <words>
       <w>arpabet</w>
       <w>backfill</w>
+      <w>cashapp</w>
       <w>cheermote</w>
+      <w>crockford</w>
+      <w>culted</w>
+      <w>dalle</w>
+      <w>downvote</w>
       <w>fakeyou</w>
+      <w>ffprobe</w>
+      <w>greenlight</w>
+      <w>greenlighting</w>
+      <w>greenlit</w>
+      <w>hanashi</w>
       <w>hifigan</w>
+      <w>replayable</w>
+      <w>retryably</w>
       <w>scrollback</w>
+      <w>softvc</w>
+      <w>symphonia</w>
+      <w>tacotron</w>
+      <w>techcrunch</w>
+      <w>transcoded</w>
+      <w>transcodings</w>
+      <w>upserting</w>
+      <w>vocoded</w>
       <w>vocoder</w>
       <w>vocoders</w>
+      <w>vocodes</w>
     </words>
   </dictionary>
 </component>

Cargo.toml

@@ -1,29 +1,67 @@
 [workspace]
 
 members = [
+  # Types: common strongly-typed tokens, ids, payloads, etc.
+  # Also common cloud buckets and other "strongly typed" configs.
+  "crates/_types/buckets",
+  "crates/_types/enums",
+  "crates/_types/reusable_types", # TODO(bt, 2022-12-21): Deprecate
+  "crates/_types/tokens",
+
+  # CLI tools for operations, etc.
+  "crates/cli/database_backup",
+  "crates/cli/database_cleanup",
+  "crates/cli/scrape_test",
+
   # Shared libraries
-  "crates/lib/config",
-  "crates/lib/container_common",
-  "crates/lib/database_queries",
-  "crates/lib/google_drive_common",
+  "crates/lib/actix_helpers",
+  "crates/lib/caching/memory_caching",
+  "crates/lib/caching/redis_caching",
+  "crates/lib/clients/cloud_storage",
+  "crates/lib/clients/fakeyou_client",
+  "crates/lib/clients/google_drive_common",
+  "crates/lib/clients/twitch_common",
+  "crates/lib/concurrency",
+  "crates/lib/config", # TODO(bt, 2023-02-13): Deprecate or repurpose
+  "crates/lib/container_common", # TODO(bt, 2023-02-13): Pull apart and rehome pieces
+  "crates/lib/crockford",
+  "crates/lib/database_queries", # TODO(bt2, 2023-02-13): Rename "mysql_queries", though this will not live beyond a single DB
+  "crates/lib/datetimes",
+  "crates/lib/errors",
+  "crates/lib/files",
+  "crates/lib/hashing",
   "crates/lib/http_server_common",
+  "crates/lib/idempotency",
   "crates/lib/jobs_common",
-  "crates/lib/memory_caching",
+  "crates/lib/media",
   "crates/lib/redis_common",
-  "crates/lib/reusable_types",
-  "crates/lib/storage_buckets_common",
+  "crates/lib/sqlite_queries",
+  "crates/lib/sqlx_mysql_helpers",
+  "crates/lib/subprocess_common",
   "crates/lib/tts_common",
-  "crates/lib/twitch_common",
+  "crates/lib/url_config",
   "crates/lib/user_input_common",
+  "crates/lib/web_scrapers",
 
-  # Application 
-  "crates/service/generic_download_job",
-  "crates/service/storyteller_web",
-  "crates/service/tts_download_job",
-  "crates/service/tts_inference_job",
-  "crates/service/twitch_pubsub_subscriber",
-  "crates/service/w2l_download_job",
-  "crates/service/w2l_inference_job",
-  "crates/service/websocket_gateway",
+  # Programs: Desktop apps
+  "crates/programs/aichatbot-sidecar",
+
+  # Services: API servers, jobs, workers, etc.
+  "crates/service/job/analytics_job",
+  "crates/service/job/download_job",
+  "crates/service/job/inference_job",
+  "crates/service/job/tts_download_job",
+  "crates/service/job/tts_inference_job",
+  "crates/service/job/w2l_download_job",
+  "crates/service/job/w2l_inference_job",
+  "crates/service/other/twitch_pubsub_subscriber",
+  "crates/service/web/dummy_service",
+  "crates/service/web/image_generation_service",
+  "crates/service/web/storyteller_web",
+  "crates/service/web/websocket_gateway",
+
+  # Service Plugins: complete subsystems that can be plugged into services (users, billing, etc.)
+  "crates/service/plugins/billing",
+  "crates/service/plugins/inference_workloads",
+  "crates/service/plugins/users",
 ]
-

Dockerfile

@@ -1,70 +1,104 @@
-# ==================== Python Build Base ====================
-
-FROM ghcr.io/storytold/docker-base-images-nvidia-cuda-experimental:080f96bc7087 as pybuild-base
-
-# The downloader needs youtube-dl as a python library
-# NB: No longer maintained: https://news.ycombinator.com/item?id=28319624
-#RUN pip3 install --upgrade youtube-dl
-
-# Used by the downloader script.
-# And it looks like youtube-dl is dead...
-# Per the README, it looks like pip will always pull the latest version.
-# https://github.com/yt-dlp/yt-dlp#update
-RUN python3 -m pip install --upgrade yt-dlp
-
-# ==================== Python Build Step 2: Wav2Lip Requirements ====================
-
-FROM pybuild-base as pybuild-requirements
-
-WORKDIR /
-
-COPY models/Wav2Lip ./models/Wav2Lip
-WORKDIR models/Wav2Lip
-
-# NB: We need Python3.6
-RUN python3.6 -m venv python
-#RUN virtualenv -p /usr/bin/python3.6  ... wait
-RUN . python/bin/activate \
-  && pip install --upgrade pip \
-  && pip install -r requirements.txt \
-  && deactivate
-
-WORKDIR /
-
-COPY models/tts ./models/tts
-WORKDIR models/tts
-
-# NB: Not sure if we need Python3.6 for Tacotron2, but Python3.8 gave me trouble.
-# NB: Setuptools fix: https://github.com/tensorflow/tensorflow/issues/34302#issuecomment-554450289
-RUN python3.6 -m venv python
-RUN . python/bin/activate \
-  && pip install --upgrade pip setuptools \
-  && pip install -r requirements-tacotron-python36.txt \
-  && deactivate
-
-# ==================== Rust Build Base ====================
+# ================================================================
+# =============== (1) set up core rust build image ===============
+# ================================================================
 
 # Custom base image
 # Make sure to add this repository so it has read acces to the base image:
 # https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access
-FROM ghcr.io/storytold/docker-base-images-rust-ssl:latest as rust-build
+# FROM ghcr.io/storytold/docker-base-images-rust-ssl:d94ce4350c3b as rust-build
+FROM ubuntu:jammy as rust-base
+
+# NB: This can be "stable" or another version.
+ARG RUST_TOOLCHAIN="1.66.0"
+
 WORKDIR /tmp
 
-COPY Cargo.lock . 
+# NB: cmake is required for freetype-sys-0.13.1, which in turn has only been added for egui.
+# NB: fontconfig is required by servo-fontconfig-sys, which is in the dependency chain for egui.
+# NB: libfontconfig-dev is required by servo-fontconfig-sys, which is in the dependency chain for egui.
+# NB: pkg-config and libssl are for container TLS; we may switch to rustls in the future.
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y \
+        build-essential \
+        cmake \
+        curl \
+        fontconfig \
+        libfontconfig1-dev \
+        libssl-dev \
+        pkg-config
+
+# NB: Fix for fontconfig (servo-fontconfig-sys): https://github.com/alacritty/alacritty/issues/4423#issuecomment-727277235
+# TODO(bt, 2023-02-23): This has not been verified to work yet.
+RUN export PKG_CONFIG_PATH=/usr/lib/x86_64-linux-gnu/pkgconfig
+
+# Install Rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
+    | sh  -s -- --default-toolchain $RUST_TOOLCHAIN -y
+
+# Install correct Rust version
+#RUN $HOME/.cargo/bin/rustup install $RUST_VERSION
+#RUN $HOME/.cargo/bin/rustup default $RUST_VERSION
+
+# Report Rust version for build debugging
+RUN $HOME/.cargo/bin/rustup show
+RUN $HOME/.cargo/bin/rustc --version
+RUN $HOME/.cargo/bin/cargo --version
+
+# Cargo Chef does Rust build caching: https://github.com/LukeMathWalker/cargo-chef
+# TODO(bt,2023-03-08): builds are failing with "no space left on device"; disabling build caching
+#RUN $HOME/.cargo/bin/cargo install cargo-chef --locked
+
+# ======================================================================
+# =============== (2) use cargo-chef to "plan" the build ===============
+# ======================================================================
+
+FROM rust-base AS planner
+
+# NB: Copying in everything does not appear to impact cached builds if irrelevant files are changed (at least at this step)
+# TODO(bt,2023-03-08): builds are failing with "no space left on device"; disabling build caching
+#COPY . .
+# TODO(bt,2023-03-08): builds are failing with "no space left on device"; disabling build caching
+#RUN $HOME/.cargo/bin/cargo chef prepare --recipe-path recipe.json
+
+# ======================================================================================================
+# =============== (3) "cook" the libraries (cacheable), then run the app build and tests ===============
+# ======================================================================================================
+
+FROM rust-base AS builder
+
+# TODO(bt,2023-03-08): builds are failing with "no space left on device"; disabling build caching
+#COPY --from=planner /tmp/recipe.json recipe.json
+
+# NB: This step builds and caches the dependencies as its own layer.
+# TODO(bt,2023-03-08): builds are failing with "no space left on device"; disabling build caching
+#RUN $HOME/.cargo/bin/cargo chef cook --release --recipe-path recipe.json
+
+# NB: Now we build and test our code.
+COPY Cargo.lock .
 COPY Cargo.toml .
-COPY sqlx-data.json .
 COPY crates/ ./crates
 COPY db/ ./db
+COPY test_data/ ./test_data
+COPY container_includes/ ./container_includes
 
-RUN $HOME/.cargo/bin/cargo fetch
-
-ARG GIT_SHA
-RUN echo -n ${GIT_SHA} > GIT_SHA
+# Print a report on disk space
+#RUN echo "Disk usage at root (before tests):"
+#RUN du -hsc / | sort -hr
+RUN echo "Disk usage at current directory (before tests):"
+RUN pwd
+RUN du -hsc * | sort -hr
 
 # Run all of the tests
-RUN SQLX_OFFLINE=true \
-  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
-  $HOME/.cargo/bin/cargo test
+#RUN SQLX_OFFLINE=true \
+#  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+#  $HOME/.cargo/bin/cargo test
+
+# Print a report on disk space
+#RUN echo "Disk usage at root (after tests):"
+#RUN du -hsc / | sort -hr
+RUN echo "Disk usage at current directory (after tests):"
+RUN pwd
+RUN du -hsc * | sort -hr
 
 # Build all the binaries.
 RUN SQLX_OFFLINE=true \
@@ -73,6 +107,30 @@ RUN SQLX_OFFLINE=true \
   --release \
   --bin storyteller-web
 
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin dummy-service
+
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin analytics-job
+
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin download-job
+
+RUN SQLX_OFFLINE=true \
+  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
+  $HOME/.cargo/bin/cargo build \
+  --release \
+  --bin inference-job
+
 RUN SQLX_OFFLINE=true \
   LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
   $HOME/.cargo/bin/cargo build \
@@ -85,12 +143,6 @@ RUN SQLX_OFFLINE=true \
   --release \
   --bin w2l-download-job
 
-RUN SQLX_OFFLINE=true \
-  LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
-  $HOME/.cargo/bin/cargo build \
-  --release \
-  --bin generic-download-job
-
 RUN SQLX_OFFLINE=true \
   LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
   $HOME/.cargo/bin/cargo build \
@@ -115,72 +167,61 @@ RUN SQLX_OFFLINE=true \
   --release \
   --bin twitch-pubsub-subscriber
 
+# Print a report on disk space
+RUN echo "Disk usage at current directory (after all builds):"
+RUN pwd
+RUN du -hsc * | sort -hr
+
+# =============================================================
+# =============== (4) construct the final image ===============
+# =============================================================
+
 # Final image
-#  FROM ubuntu:xenial
-FROM pybuild-requirements as final
+FROM ubuntu:jammy as final
+
+# See: https://github.com/opencontainers/image-spec/blob/master/annotations.md
+LABEL org.opencontainers.image.title='Storyteller Rust'
+LABEL org.opencontainers.image.authors='bt@brand.io, echelon@gmail.com'
+LABEL org.opencontainers.image.description='All of the binaries from the Rust monorepo'
+LABEL org.opencontainers.image.documentation='https://github.com/storytold/storyteller-web'
+LABEL org.opencontainers.image.source='https://github.com/storytold/storyteller-web'
+LABEL org.opencontainers.image.url='https://github.com/storytold/storyteller-web'
 
 WORKDIR /
 
-COPY scripts/ ./scripts
-
-# See: https://github.com/opencontainers/image-spec/blob/master/annotations.md
-LABEL org.opencontainers.image.authors='bt@brand.io, echelon@gmail.com'
-LABEL org.opencontainers.image.url='https://github.com/storytold/storyteller-web'
-LABEL org.opencontainers.image.documentation='https://github.com/storytold/storyteller-web'
-LABEL org.opencontainers.image.source='https://github.com/storytold/storyteller-web'
-
-# NB(echelon,2022-05-31): NVIDIA revoked some of their keys.
-# A better long term fix is to remove these dependencies outright and not have ML deps in this repo.
-# https://developer.nvidia.com/blog/updating-the-cuda-linux-gpg-repository-key/
-# https://forums.developer.nvidia.com/t/invalid-public-key-for-cuda-apt-repository/212901/11
-RUN apt-key del 7fa2af80
-
-# NB: Fix for 'E: Conflicting values set for option Signed-By regarding source'
-# NB: We need to remove all sources
-RUN rm $(find /etc/apt | grep cuda)
-
-#RUN wget https://developer.download.nvidia.com/compute/cuda/repos/$distro/$arch/cuda-keyring_1.0-1_all.deb
-RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb
-RUN dpkg -i cuda-keyring_1.0-1_all.deb
-
-# NB: Comment this out for non-debug images
-# TODO: Figure out how this is done elsewhere with just the static binaries
-# Others: `mysql-client`
-RUN apt-get update \
-    && apt-get install -y \
-        curl \
-        python3-pip \
-        python3 \
-        wget
-
-RUN python3 --version
-
-# TODO: Use venv to do this instead.
-RUN pip3 install gdown
+# Give the container its version so it can report over HTTP.
+ARG GIT_SHA
+RUN echo -n ${GIT_SHA} > GIT_SHA
 
 # Copy all the binaries.
-COPY --from=rust-build /tmp/target/release/storyteller-web /
-COPY --from=rust-build /tmp/target/release/tts-download-job /
-COPY --from=rust-build /tmp/target/release/tts-inference-job /
-COPY --from=rust-build /tmp/target/release/w2l-download-job /
-COPY --from=rust-build /tmp/target/release/w2l-inference-job /
-COPY --from=rust-build /tmp/target/release/websocket-gateway /
-COPY --from=rust-build /tmp/target/release/twitch-pubsub-subscriber /
-COPY --from=rust-build /tmp/target/release/generic-download-job /
-COPY --from=rust-build /tmp/GIT_SHA /
+COPY --from=builder /tmp/target/release/storyteller-web /
+COPY --from=builder /tmp/target/release/dummy-service /
+COPY --from=builder /tmp/target/release/analytics-job /
+COPY --from=builder /tmp/target/release/download-job /
+COPY --from=builder /tmp/target/release/inference-job /
+COPY --from=builder /tmp/target/release/tts-download-job /
+COPY --from=builder /tmp/target/release/tts-inference-job /
+COPY --from=builder /tmp/target/release/w2l-download-job /
+COPY --from=builder /tmp/target/release/w2l-inference-job /
+COPY --from=builder /tmp/target/release/websocket-gateway /
+COPY --from=builder /tmp/target/release/twitch-pubsub-subscriber /
 
 # SSL certs are required for crypto
-COPY --from=rust-build /etc/ssl /etc/ssl
+COPY --from=builder /etc/ssl /etc/ssl
 
 # Required dynamically linked libraries
-COPY --from=rust-build /lib/x86_64-linux-gnu/libssl.*             /lib/x86_64-linux-gnu/
-COPY --from=rust-build /lib/x86_64-linux-gnu/libcrypto.*          /lib/x86_64-linux-gnu/
+COPY --from=builder /usr/lib/x86_64-linux-gnu/libssl.*             /lib/x86_64-linux-gnu/
+COPY --from=builder /usr/lib/x86_64-linux-gnu/libcrypto.*          /lib/x86_64-linux-gnu/
+
+# Container includes
+COPY container_includes/ /container_includes
 
 # Make sure all the links resolve
 RUN ldd storyteller-web
 
 # Without a .env file, Rust crashes "mysteriously" (ugh)
 RUN touch .env
+RUN touch .env-download-job
 RUN touch .env-secrets
 
 EXPOSE 8080

README.md

@@ -1,77 +1,46 @@
 storyteller-rust
 ================
 
-Our Rust monorepo, which services our 
-API surface area. 
-
-Application overview
---------------------
-
-* [FakeYou.com](https://fakeyou.com) is powered by:
-  * HTTP APIs:
-    * `storyteller-web` - HTTP + database monolith for FakeYou
-  * Asynchronous Jobs:
-    * `tts-download-job` - Async download of models
-    * `tts-inference-job` - Async TTS inference
-    * `w2l-download-job` - Async download of videos and images
-    * `w2l-inference-job` - Async W2L inference
-
-* [StorytellerStream](https://storyteller.stream) is powered by:
-  * HTTP APIs:
-    * `storyteller-web` - HTTP + database monolith for FakeYou
-  * Websocket events:
-    * `websocket-gateway` - Websocket to pull down Twitch events.
-  * Twitch API integration:
-    * `twitch-pubsub-subscriber` - Dynamically subscribes to Twitch PubSub for "active" users
-
-Schema and API notes
---------------------
-
-* [Redis Schema Notes README](_docs/redis_schema.md)
-
-Local development
------------------
-
-[Local development setup README](_docs/local_development_setup.md)
-
-Production and Deployment
--------------------------
-
-[Production builds and deployment README](_docs/production_builds_and_deployment.md)
-
-Development notes
------------------
-
-[Development notes README](_docs/development_notes.md)
-
-Dump a MySQL report
--------------------
+Our Rust monorepo, which serves both our API surface area and our downloadable client applications.
 
 ```
-mysql -u storyteller \
-  -pPasswordHere \
-  -h IP_HERE \
-  -P 3306 \
-  -D storyteller \
-  -B \
-  -e "select * from voice_clone_requests;" | sed "s/'/\'/;s/\t/\",\"/g;s/^/\"/;s/$/\"/;s/\n//g" > voice_clone_report.csv
-```
-
-TODO
-----
-
-* Revise this README to be more useful.
-
-Notes / TODOs:
-
-* Examples for good Actix+Sqlx Tests:
-  https://stackoverflow.com/questions/65370752/how-do-i-create-an-actix-web-server-that-accepts-both-sqlx-database-pools-and-tr
-
-* Actix/sqlx runtime compat:
-  https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734
-  
-* Redis caching of more endpoints
-
-* Jobs for analytics queries (#uses per model, etc)
-* Jobs for ingesting Discord, Twitter, Reddit RPAN chat, Twitch chat (not PubSub), etc.
-
+                 ."-,.__
+                 `.     `.  ,
+              .--'  .._,'"-' `.
+             .    .'         `'
+             `.   /          ,'
+               `  '--.   ,-"'
+                `"`   |  \
+                   -. \, |
+                    `--Y.'      ___.
+                         \     L._, \
+               _.,        `.   <  <\                _
+             ,' '           `, `.   | \            ( `
+          ../, `.            `  |    .\`.           \ \_
+         ,' ,..  .           _.,'    ||\l            )  '".
+        , ,'   \           ,'.-.`-._,'  |           .  _._`.
+      ,' /      \ \        `' ' `--/   | \          / /   ..\
+    .'  /        \ .         |\__ - _ ,'` `        / /     `.`.
+    |  '          ..         `-...-"  |  `-'      / /        . `.
+    | /           |L__           |    |          / /          `. `.
+   , /            .   .          |    |         / /             ` `
+  / /          ,. ,`._ `-_       |    |  _   ,-' /               ` \
+ / .           \"`_/. `-_ \_,.  ,'    +-' `-'  _,        ..,-.    \`.
+.  '         .-f    ,'   `    '.       \__.---'     _   .'   '     \ \
+' /          `.'    l     .' /          \..      ,_|/   `.  ,'`     L`
+|'      _.-""` `.    \ _,'  `            \ `.___`.'"`-.  , |   |    | \
+||    ,'      `. `.   '       _,...._        `  |    `/ '  |   '     .|
+||  ,'          `. ;.,.---' ,'       `.   `.. `-'  .-' /_ .'    ;_   ||
+|| '              V      / /           `   | `   ,'   ,' '.    !  `. ||
+||/            _,-------7 '              . |  `-'    l         /    `||
+. |          ,' .-   ,' ||               | .-.        `.      .'     ||
+ `'        ,'    `".'    |               |    `.        '. -.'       `'
+          /      ,'      |               |,'    \-.._,.'/'
+          .     /        .               .       \    .''
+        .`.    |         `.             /         :_,'.'
+          \ `...\   _     ,'-.        .'         /_.-'                       S T O R Y T E L L E R
+           `-.__ `,  `'   .  _.>----''.  _  __  /                        Turning memes into dreams
+                .'        /"'          |  "'   '_                      One Rust Monorepo at a time
+               /_|.-'\ ,".             '.'`__'-( \
+                 / ,"'"\,'               `/  `-.|" 
+```

_develop/docs/aichatbot_sidecar.md

@@ -0,0 +1,38 @@
+AI Chatbot Sidecar
+==================
+
+This is the first app that doesn't run as a part of our SoA API architecture. (Though it is written such that we 
+could port it.) This app is responsible for generating all the content that goes into our streaming AI-powered TV
+programs.
+
+### Windows Installation
+
+The application uses sqlite, which is tricky to install on Windows.
+
+1. Put together an sqlite build directory containing the sources and dll file. One is checked in under this monorepo 
+   in `./windows/sqlite` at the time of this writing, though that may change in the future.
+
+
+2. Add environment variables for the sqlite build directory (see https://github.com/diesel-rs/diesel/issues/487)
+
+    * PATH (append the sqlite build directory to the end)
+    * SQLITE3_LIB_DIR (a new environment variable; set it to the sqllite build path)
+
+
+3. Build the `sqlite3.lib` file with the following:
+
+   ```
+   lib /def:sqlite3.def /out:sqlite3.lib /MACHINE:amd64 
+   ```
+   
+   This must be done in "Developer Command Prompt for VS 2022" (or similar), not `cmd` or `PowerShell`.
+
+
+4. Run the diesel_cli tool build  
+   ```
+   cargo install diesel_cli --no-default-features --features sqlite
+   ```
+
+   If it still complains about the `sqlite3.lib` file not being found, be sure to double check that `echo %PATH%` shows 
+   the updated PATH from the steps above.
+

_develop/docs/local_development_k8s.md

@@ -0,0 +1,20 @@
+# Local Kubernetes environment for standing up the APIs and their dependencies
+
+- Install minikube, kubectl, docker, and musl from your OS package manager
+- `rustup target add x86_64-unknown-linux-musl`
+- `cargo build --release --bin storyteller-web --target=x86_64-unknown-linux-musl`
+- `minikube start --extra-config=apiserver.service-node-port-range=1-65535`
+- `eval $(minikube docker-env)`
+- `minikube mount /path/to/storyteller-rust/:/storyteller-rust &> /dev/null &`
+- `docker build -t storyteller-web -f localdev/Dockerfile .`
+- `kubectl apply -f localdev/kubernetes.yml`
+- Get the ip address of minikube with `minikube ip`, this is where all the services will be running on their 
+regular ports (12345 for the api, 6379 for redis, 3306 for mysql)
+- That's it, we just spun up mysql, applied database migrations, spun up redis, and spun up our api
+
+# Development tips
+- Rebuild and "deploy" (locally) with `cargo build --release --bin storyteller-web --target=x86_64-unknown-linux-musl && kubectl rollout restart storyteller-web`
+- View logs from the api with `kubectl logs  storyteller-web<TAB>`
+- Delete everything minikube (if you start running out of disk space or what-have-you) with `minikube delete`
+- See pod status with `kubectl get pods`
+- If there are errors you can get more info with `kubectl describe pods`

_develop/docs/local_development_setup.md

@@ -1,9 +1,9 @@
 dev setup
 =========
 
-[Back to main README](../README.md)
+[Back to main README](../../README.md)
 
-### Debugging and Fixing CUDA/PyTorch
+## Debugging and Fixing CUDA/PyTorch
 
 See this spreadsheet for previous battles with version incompatibility:
 https://docs.google.com/spreadsheets/d/1BEdLmwOzo3r83-iJn9sj6co1VT92t3lIeJ-lNH25bdQ/edit#gid=0
@@ -11,27 +11,53 @@ https://docs.google.com/spreadsheets/d/1BEdLmwOzo3r83-iJn9sj6co1VT92t3lIeJ-lNH25
 Ubuntu might upgrade the driver by accident, and it might need reinstallation. Who knows.
 This stuff is a nightmare.
 
-### Database Setup
+## Database Setup
+
+### Mac Install
+
+```
+brew install mysql
+# If mysql -uroot fails, reboot the machine:
+sudo reboot now
+```
+
+### Linux Install
 
 Install the following libraries, and see the notes further below about MySQL on Ubuntu 20.04.
 
 ```
+jq # for combining mysql codegen outputs
+libmysqlclient-dev
 mysql-server
-imagemagick
+libsqlite3-dev
+pkgconf # if using openssl instead of rustls
 ```
 
+### Migrations
+
 To manage the database and perform migrations, install the Rust tools diesel and sqlx.
 
 We'll be using diesel to manage the migrations, but sqlx within the app to actually perform queries.
 Diesel is an ORM, which is dumb, so we use sqlx as at-compile-time typesafe SQL.
 
+The CLI is necessary to perform codegen:
+
 ```
 sudo apt-get install libmysqlclient-dev
-cargo install sqlx-cli --no-default-features --features mysql [2022-01-16: is this needed?]
+cargo install sqlx-cli --no-default-features --features rustls,mysql,sqlite
 ```
 
-To install a more modern version of Diesel (unreleased) that 
-supports the "migrations_dir" toml config option:
+Diesel now supports configuring the migration directory via an environment variable,
+so we can install the currently published version:
+
+```
+cargo install diesel_cli \
+  --no-default-features \
+  --features mysql,sqlite
+```
+
+If the modern version fails, try installing a cherry-picked version with a 
+TOML-based migration directory configuration:
 
 ```
 cargo install diesel_cli \
@@ -60,6 +86,21 @@ If MySql in local dev can't be connected to, reset the accounts:
 
 https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux
 
+#### MySql on Ubuntu 22.04
+
+Should work largely out of the box. We'll need a dev account:
+
+```
+# Connect to mysql:
+# sudo mysql -u root -p (password is "root")
+
+use mysql;
+CREATE USER 'storyteller'@'localhost' IDENTIFIED BY 'password';
+GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';
+```
+
+Then verify with `./dev_mysql_connect.sh`
+
 #### Fixing dev MySql on Ubuntu 20.04
 
 For some reason, the MySql default install on 20.04 gave me a bunch of trouble.

_develop/docs/old_readme.md

@@ -0,0 +1,77 @@
+old readme
+==========
+
+Contents of the main README.md, which will always be woefully out of date. Docs will migrate into this folder on a 
+per-subject basis. App-specific docs will live under each app.
+
+Application overview
+--------------------
+
+* [FakeYou.com](https://fakeyou.com) is powered by:
+    * HTTP APIs:
+        * `storyteller-web` - HTTP + database monolith for FakeYou
+    * Asynchronous Jobs:
+        * `tts-download-job` - Async download of models
+        * `tts-inference-job` - Async TTS inference
+        * `w2l-download-job` - Async download of videos and images
+        * `w2l-inference-job` - Async W2L inference
+
+* [StorytellerStream](https://storyteller.stream) is powered by:
+    * HTTP APIs:
+        * `storyteller-web` - HTTP + database monolith for FakeYou
+    * Websocket events:
+        * `websocket-gateway` - Websocket to pull down Twitch events.
+    * Twitch API integration:
+        * `twitch-pubsub-subscriber` - Dynamically subscribes to Twitch PubSub for "active" users
+
+Schema and API notes
+--------------------
+
+* [Redis Schema Notes README](_docs/redis_schema.md)
+
+Local development
+-----------------
+
+[Local development setup README](_docs/local_development_setup.md)
+
+Production and Deployment
+-------------------------
+
+[Production builds and deployment README](_docs/production_builds_and_deployment.md)
+
+Development notes
+-----------------
+
+[Development notes README](_docs/development_notes.md)
+
+Dump a MySQL report
+-------------------
+
+```
+mysql -u storyteller \
+  -pPasswordHere \
+  -h IP_HERE \
+  -P 3306 \
+  -D storyteller \
+  -B \
+  -e "select * from voice_clone_requests;" | sed "s/'/\'/;s/\t/\",\"/g;s/^/\"/;s/$/\"/;s/\n//g" > voice_clone_report.csv
+```
+
+TODO
+----
+
+* Revise this README to be more useful.
+
+Notes / TODOs:
+
+* Examples for good Actix+Sqlx Tests:
+  https://stackoverflow.com/questions/65370752/how-do-i-create-an-actix-web-server-that-accepts-both-sqlx-database-pools-and-tr
+
+* Actix/sqlx runtime compat:
+  https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734
+
+* Redis caching of more endpoints
+
+* Jobs for analytics queries (#uses per model, etc)
+* Jobs for ingesting Discord, Twitter, Reddit RPAN chat, Twitch chat (not PubSub), etc.
+

_develop/localdev/Dockerfile

@@ -0,0 +1,23 @@
+FROM rust:latest
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -y \
+        mariadb-client \
+        imagemagick  \
+        libmariadb-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+
+RUN cargo install diesel_cli --features mysql && \
+  cargo install sqlx-cli --features mysql
+
+WORKDIR /storyteller-rust
+
+ARG GIT_SHA
+
+RUN echo -n ${GIT_SHA} > /GIT_SHA
+
+CMD ["/storyteller-rust/localdev/startup.sh"]
+

_develop/localdev/kubernetes.yml

@@ -0,0 +1,153 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata: 
+  name: mysql
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mysql
+  template:
+    metadata:
+      labels:
+        app: mysql
+    spec:
+      containers:
+      - name: mysql
+        image: mysql:8.0.30
+        env:
+        - name: MYSQL_USER
+          value: storyteller
+        - name: MYSQL_PASSWORD
+          value: password
+        - name: MYSQL_DATABASE
+          value: storyteller
+        - name: MYSQL_ROOT_PASSWORD
+          value: password
+        ports:
+          - containerPort: 3306 
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata: 
+  name: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+      - name: redis
+        image: redis:7.0.4
+        ports:
+          - containerPort: 6379
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata: 
+  name: storyteller-web
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: storyteller-web
+  template:
+    metadata:
+      labels:
+        app: storyteller-web
+    spec:
+      containers:
+      - name: storyteller-web
+        image: storyteller-web:latest
+        imagePullPolicy: Never
+        env:
+          - name: MYSQL_URL
+            value: mysql://storyteller:password@mysql.default/storyteller
+          - name: REDIS_0_URL
+            value: redis://redisd@redis.default/0
+          - name: REDIS_1_URL
+            value: redis://redisd@redis.default/1
+          - name: ACCESS_KEY
+            value: garbage
+          - name: SECRET_KEY
+            value: garbage
+          - name: REGION_NAME
+            value: garbage
+          - name: W2L_PRIVATE_DOWNLOAD_BUCKET_NAME
+            value: garbage
+          - name: W2L_PUBLIC_DOWNLOAD_BUCKET_NAME
+            value: garbage
+          - name: AUDIO_UPLOADS_BUCKET_ROOT
+            value: garbage
+          - name: TWITCH_USERNAME
+            value: garbage
+          - name: TWITCH_STREAM_KEY
+            value: garbage
+          - name: TWITCH_APP_CLIENT_ID
+            value: garbage
+          - name: TWITCH_APP_CLIENT_SECRET
+            value: garbage
+        ports:
+          - containerPort: 12345
+        volumeMounts:
+          - mountPath: /storyteller-rust
+            name: storyteller-rust-folder
+      volumes:
+        - name: storyteller-rust-folder
+          hostPath:
+            path: /storyteller-rust
+---
+
+apiVersion: v1
+kind: Service
+metadata: 
+  name: mysql
+spec:
+  type: NodePort
+  ports:
+    - name: mysql
+      port: 3306
+      nodePort: 3306
+  selector:
+    app: mysql
+
+---
+
+apiVersion: v1
+kind: Service
+metadata: 
+  name: redis
+spec:
+  type: NodePort
+  ports:
+    - name: redis
+      port: 6379
+      nodePort: 6379
+  selector:
+    app: redis
+
+---
+
+apiVersion: v1
+kind: Service
+metadata: 
+  name: storyteller-web
+spec:
+  type: NodePort
+  ports:
+    - name: storyteller-web
+      port: 12345
+      nodePort: 12345
+  selector:
+    app: storyteller-web
+
+

_develop/localdev/nginx-http-config/storyteller-ssl

@@ -0,0 +1,258 @@
+# Nginx configs for "jungle.horse" and "api.jungle.horse" (http)
+#
+# On Ubuntu Linux, this config is located at the following location:
+# 
+#  - /etc/nginx/sites-enabled/{name-of-site}
+#
+# Move this file there, and then restart with,
+#
+#   sudo service nginx restart
+#
+# On Mac, this is located at the following locations:
+#
+#  - /opt/homebrew/etc/nginx/servers/{name-of-site} (for nginx installed with homebrew)
+#  - /usr/local/etc/nginx/sites-enabled/{name-of-site} (for default nginx install)
+#
+#
+# On Mac, verify configs and restart with,
+#
+#   sudo nginx -t
+#   sudo nginx -s reload
+#
+#           ================
+#           SSL Instructions
+#           ================
+#
+# You can launch Chrome in a way it disregards self-signed certs and CORS:
+#
+#     chromium  --disable-web-security --ignore-certificate-errors --user-data-dir=~/chromeTemp
+#
+# To create self-signed certs on Ubuntu Linux,
+#
+#     sudo apt-get install openssl
+#
+#     sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/localhost.key -out /etc/ssl/certs/localhost.crt
+#
+#     ^ most fields can be blank, but the "Common Name" should be `*.jungle.horse`
+#
+#     sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/devfakeyou.key -out /etc/ssl/certs/devfakeyou.crt
+#     sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/devstoryteller.key -out /etc/ssl/certs/devstoryteller.crt
+#
+#     ^ most fields can be blank, but the "Common Name" should be `*.dev.fakeyou.com` and `*.dev.storyteller.io` respectively.
+#
+# For more: https://www.cloudsavvyit.com/1306/how-to-create-and-use-self-signed-ssl-on-nginx/
+#
+# To create self-signed certs on Mac,
+#
+#     openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt
+#
+#     ^ most fields can be blank, but the "Common Name" should be `*.jungle.horse`
+#
+# On Mac, load the self signed cert into the keychain with the following command:
+#
+#     sudo security add-trusted-cert \
+#      -d -r trustRoot \
+#      -k /Library/Keychains/System.keychain /etc/ssl/certs/localhost.crt
+#
+# Self signed certs might be considered insecure by the browser. In Firefox, 
+# set about:config `security.tls.insecure_fallback_hosts` to include a list of
+# permitted domains. You may also need to configure other settings. 
+#
+#
+
+# ========================================================
+#                      FAKEYOU
+# ========================================================
+
+# FakeYou javascript frontend over plain HTTP (launched by Yarn)
+server {
+    listen 80;
+    server_name dev.fakeyou.com;
+    location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:7000;
+        proxy_redirect off;
+    }
+}
+
+# FakeYou API backend over plain HTTP (Rust app)
+server {
+    listen 80;
+    server_name api.dev.fakeyou.com;
+    location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:12345;
+        proxy_redirect off;
+    }
+}
+
+# FakeYou javascript frontend over SSL (launched by Yarn)
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    ssl_certificate /etc/ssl/certs/devfakeyou.crt;
+    ssl_certificate_key /etc/ssl/private/devfakeyou.key;
+    ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
+    server_name dev.fakeyou.com;
+    location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:7000;
+        proxy_redirect off;
+    }
+}
+
+# FakeYou API backend over SSL (Rust app)
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    ssl_certificate /etc/ssl/certs/devfakeyou.crt;
+    ssl_certificate_key /etc/ssl/private/devfakeyou.key;
+    ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
+    server_name api.dev.fakeyou.com;
+    location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:12345;
+        proxy_redirect off;
+    }
+}
+
+# ========================================================
+#                      STORYTELLER
+# ========================================================
+
+# Storyteller javascript frontend over plain HTTP (launched by Yarn)
+server {
+    listen 80;
+    server_name dev.storyteller.io;
+    location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:7000;
+        proxy_redirect off;
+    }
+}
+
+# Storyteller API backend over plain HTTP
+server {
+    listen 80;
+    server_name api.dev.storyteller.io;
+    location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:12345;
+        proxy_redirect off;
+    }
+}
+
+# Storyteller javascript frontend over SSL (launched by Yarn)
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    ssl_certificate /etc/ssl/certs/devstoryteller.crt;
+    ssl_certificate_key /etc/ssl/private/devstoryteller.key;
+    ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
+    server_name dev.storyteller.io;
+    location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:7000;
+        proxy_redirect off;
+    }
+}
+
+# Storyteller API backend over SSL
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    ssl_certificate /etc/ssl/certs/devstoryteller.crt;
+    ssl_certificate_key /etc/ssl/private/devstoryteller.key;
+    ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
+    server_name api.dev.storyteller.io;
+    location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:12345;
+        proxy_redirect off;
+    }
+}
+
+# ========================================================
+#            OLD PRODUCTS THAT NEED UPDATING
+# ========================================================
+
+#     ## TODO(2022-02-13): This is new and untested
+#     # WebSocket API backend over plain HTTP
+#     server {
+#         listen 80;
+#         server_name ws.jungle.horse;
+#         location / {
+#             proxy_set_header Host $host;
+#             proxy_pass http://127.0.0.1:54321;
+#             proxy_redirect off;
+#         }
+#     }
+#
+#     # Websocket API backend over SSL
+#     server {
+#         listen 443 ssl;
+#         listen [::]:443 ssl;
+#         ssl_certificate /etc/ssl/certs/localhost.crt;
+#         ssl_certificate_key /etc/ssl/private/localhost.key;
+#         ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
+#         server_name ws.jungle.horse;
+#         location / {
+#             proxy_set_header Host $host;
+#             proxy_pass http://127.0.0.1:54321;
+#             proxy_redirect off;
+#         }
+#     }
+#
+#     # Javascript frontend over plain HTTP
+#     server {
+#         listen 80;
+#         server_name jungle.horse;
+#         location / {
+#             proxy_set_header Host $host;
+#             proxy_pass http://127.0.0.1:7000;
+#             proxy_redirect off;
+#         }
+#     }
+#
+#     # TODO(2022-02-13): This is new and untested
+#     # Javascript frontend (OBS Layer) over plain HTTP
+#     server {
+#         listen 80;
+#         server_name obs.jungle.horse;
+#         location / {
+#             proxy_set_header Host $host;
+#             proxy_pass http://127.0.0.1:7007;
+#             proxy_redirect off;
+#         }
+#     }
+#
+#     # Javascript frontend over SSL
+#     server {
+#         listen 443 ssl;
+#         listen [::]:443 ssl;
+#         ssl_certificate /etc/ssl/certs/localhost.crt;
+#         ssl_certificate_key /etc/ssl/private/localhost.key;
+#         ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
+#         server_name jungle.horse;
+#         location / {
+#             proxy_set_header Host $host;
+#             proxy_pass http://127.0.0.1:7000;
+#             proxy_redirect off;
+#         }
+#     }
+#
+#     # Javascript frontend over SSL (OBS Layer)
+#     server {
+#         listen 443 ssl;
+#         listen [::]:443 ssl;
+#         ssl_certificate /etc/ssl/certs/localhost.crt;
+#         ssl_certificate_key /etc/ssl/private/localhost.key;
+#         ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
+#         server_name obs.jungle.horse;
+#         location / {
+#             proxy_set_header Host $host;
+#             proxy_pass http://127.0.0.1:7007;
+#             proxy_redirect off;
+#         }
+#     }
+#

_develop/localdev/startup.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set +e +o pipefail
+diesel migration run --database-url="${MYSQL_URL}"
+set -e -o pipefail
+./target/x86_64-unknown-linux-musl/release/storyteller-web
+

_develop/localdev/ubuntu_setup.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+echo 'Installing APT packages...'
+
+# OpenSSL requirements: libssl-dev, pkg-config
+# Database requirements: mysql-server, libmysqlclient-dev
+# Other tools: jq (db codegen)
+sudo apt-get install -y \
+  jq \
+  libmysqlclient-dev \
+  libssl-dev \
+  mysql-server \
+  pkg-config \
+  --no-install-recommends
+
+echo 'Installing Cargo packages...'
+
+# Install Diesel CLI for migrations
+cargo install diesel_cli \
+  --no-default-features \
+  --features mysql
+
+# Install SQLx for codegen
+cargo install sqlx-cli \
+  --no-default-features \
+  --features rustls,mysql
+
+echo 'Installing Database user...'
+
+# Create DB account (using default system mysql password)
+sudo mysql \
+  -u "root" \
+  -p"password" \
+  -e "use mysql; CREATE DATABASE IF NOT EXISTS storyteller; CREATE USER IF NOT EXISTS 'storyteller'@'localhost' IDENTIFIED BY 'password'; GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';"
+
+

_migrations/2021-05-15-071534_users/up.sql

@@ -1,235 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-CREATE TABLE users (
-  -- Not used for anything except replication.
-  id BIGINT(20) NOT NULL AUTO_INCREMENT,
-
-  -- Visible "primary key"
-  token VARCHAR(32) NOT NULL,
-
-  -- Username is a lookup key; display_name allows the user to add custom case.
-  username VARCHAR(20) NOT NULL,
-  display_name VARCHAR(20) NOT NULL,
-
-  email_address VARCHAR(255) NOT NULL,
-  email_confirmed BOOLEAN NOT NULL DEFAULT false,
-
-  -- The role assigned to the user confers permissions.
-  user_role_slug VARCHAR(16) NOT NULL,
-
-  -- ========== CREDENTIALS ==========
-
-  -- Bcrypt password hash. Granted, there are newer methods:
-  -- https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html
-  -- Passwords may be a max of 64 characters per BCrypt.
-  password_hash BINARY(60) NOT NULL,
-
-  -- Incremented with every update to the password.
-  password_version INT NOT NULL DEFAULT 0,
-
-  -- ========== ABUSE TRACKING ==========
-
-  -- For abuse tracking.
-  -- Wide enough for IPv4/6
-  ip_address_creation VARCHAR(40) NOT NULL,
-  ip_address_last_login VARCHAR(40) NOT NULL,
-  ip_address_last_update VARCHAR(40) NOT NULL,
-
-  -- ========== DISPLAY / PROFILE ==========
-
-  -- Gravatar image hashes are precomputed.
-  email_gravatar_hash CHAR(32) NOT NULL,
-
-  -- The profile of the user in markdown (user editable).
-  profile_markdown TEXT NOT NULL,
-
-  -- Generated HTML (not user-editable).
-  profile_rendered_html TEXT NOT NULL,
-
-  -- An uploaded avatar. Public hash in our bucket.
-  -- avatar_public_bucket_hash CHAR(32) DEFAULT NULL,
-
-  -- ========== SOCIAL MEDIA ==========
-
-  -- Social media usernames
-  -- These are not confirmed. We'll need to build an OAuth system to handle that.
-  discord_username VARCHAR(36) DEFAULT NULL,
-  twitter_username VARCHAR(36) DEFAULT NULL,
-  twitch_username VARCHAR(36) DEFAULT NULL,
-  patreon_username VARCHAR(36) DEFAULT NULL,
-  github_username VARCHAR(36) DEFAULT NULL,
-  cashapp_username VARCHAR(36) DEFAULT NULL,
-  website_url VARCHAR(255) DEFAULT NULL,
-
-  -- ========== USER PREFERENCES ==========
-
-  -- If the user doesn't want to use gravatar and doesn't have an uploaded avatar.
-  disable_gravatar BOOLEAN NOT NULL DEFAULT false,
-
-  -- (THIS MIGHT NOT BE USED)
-  -- NB: DO NOT SORT!
-  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
-  -- Visibility preference used at TTS inference time (which can be changed on the result itself).
-  -- Moderators will still see them.
-  preferred_tts_result_visibility ENUM(
-    'public',
-    'hidden',
-    'private'
-  ) NOT NULL DEFAULT 'public',
-
-  -- (THIS MIGHT NOT BE USED)
-  -- NB: DO NOT SORT!
-  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
-  -- Visibility preference used at TTS inference time (which can be changed on the result itself).
-  -- Moderators will still see them.
-  preferred_w2l_result_visibility ENUM(
-    'public',
-    'hidden',
-    'private'
-  ) NOT NULL DEFAULT 'public',
-
-  -- Auto play preferences
-  -- Tri-state because we want to potentially change the default behavior.
-  auto_play_audio_preference BOOLEAN DEFAULT NULL,
-  auto_play_video_preference BOOLEAN DEFAULT NULL,
-
-  -- Favorite TTS voice to use by default
-  -- maybe_preferred_tts_model_token VARCHAR(32) DEFAULT NULL,
-
-  -- Favorite W2L model to use by default
-  -- maybe_preferred_w2l_template_token VARCHAR(32) DEFAULT NULL,
-
-  -- Settings
-  -- DO NOT REORDER.
-  -- dark_mode_preference ENUM(
-  --   'light-mode',
-  --   'dark-mode',
-  --   'use-clock'
-  --  ) NOT NULL DEFAULT 'light-mode',
-
-  -- ========== STATS ==========
-
-  -- For tracking stats.
-  -- The "cached" values are updated by a background job.
-  -- cached_tts_rendered_counter INT(10) NOT NULL DEFAULT 0,
-  -- cached_w2l_rendered_counter INT(10) NOT NULL DEFAULT 0,
-
-  -- ========== MODERATION DETAILS ==========
-
-  -- Different than deleted.
-  -- Users still show up, but can't do anything.
-  is_banned BOOLEAN NOT NULL DEFAULT false,
-
-  -- If a moderator has comments.
-  maybe_mod_comments VARCHAR(255) DEFAULT NULL,
-
-  -- The last moderator that made changes.
-  maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
-
-  -- ========== VECTOR CLOCK ==========
-
-  -- Incremented with every update.
-  version INT NOT NULL DEFAULT 0,
-
-  -- ========== TIMESTAMPS ==========
-
-  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-
-  -- If the user is deleted, we set this.
-  -- This is different than banned. These users won't show up at all.
-  user_deleted_at TIMESTAMP NULL,
-  mod_deleted_at TIMESTAMP NULL,
-
-  -- INDICES --
-  PRIMARY KEY (id),
-  UNIQUE KEY (token),
-  UNIQUE KEY (username),
-  UNIQUE KEY (email_address),
-  KEY fk_user_role_slug (user_role_slug)
-
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
-
-CREATE TABLE user_roles (
-  -- Not used for anything except replication.
-  id BIGINT(20) NOT NULL AUTO_INCREMENT,
-
-  -- Effective "primary key"
-  slug CHAR(16) NOT NULL,
-
-  name VARCHAR(255) NOT NULL,
-
-  -- Usage
-  can_use_tts BOOLEAN NOT NULL DEFAULT FALSE,
-  can_use_w2l BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_own_tts_results BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_own_w2l_results BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_own_account BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- Contribution
-  can_upload_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
-  can_upload_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_own_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_own_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- Moderation
-  can_approve_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
-  -- Edit includes "lock_from_use" and "lock_from_modification"
-  can_edit_other_users_profiles BOOLEAN NOT NULL DEFAULT FALSE,
-  can_edit_other_users_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
-  can_edit_other_users_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_other_users_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_other_users_tts_results BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_other_users_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_other_users_w2l_results BOOLEAN NOT NULL DEFAULT FALSE,
-  can_ban_users BOOLEAN NOT NULL DEFAULT FALSE,
-  can_delete_users BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- Incremented with every update.
-  version INT NOT NULL DEFAULT 0,
-
-  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-
-  -- INDICES --
-  PRIMARY KEY (id),
-  UNIQUE KEY (slug)
-
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
-
-CREATE TABLE user_sessions (
-  -- Not used for anything except replication.
-  id BIGINT(20) NOT NULL AUTO_INCREMENT,
-
-  -- Session entropy
-  token VARCHAR(32) NOT NULL,
-
-  -- Foreign key to user
-  user_token VARCHAR(32) NOT NULL,
-
-  -- Track each session's creation IP.
-  -- Wide enough for IPv4/6
-  ip_address_creation VARCHAR(40) NOT NULL,
-
-  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-
-  -- Session termination time.
-  -- This must be set by the server code, or the session is invalid
-  expires_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-
-  -- deletion = session termination
-  -- Typically these are destroyed by users, but if in the future we allow mods to
-  -- delete them, it doesn't really matter who did the deletion: sessions are not
-  -- designed to be recoverable.
-  deleted_at TIMESTAMP NULL,
-
-  -- INDICES --
-  PRIMARY KEY (id),
-  UNIQUE KEY (token),
-  KEY fk_user_token (user_token),
-  KEY index_ip_address_creation (ip_address_creation)
-
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_migrations/2022-08-05-202017_generic_download_jobs/up.sql

@@ -1,110 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-CREATE TABLE generic_download_jobs (
-  -- Not used for anything except replication.
-  id BIGINT(20) NOT NULL AUTO_INCREMENT,
-
-  -- Effective "primary key" (PUBLIC)
-  -- This is so the in-progress results can be looked up by the UI.
-  token VARCHAR(32) NOT NULL,
-
-  -- Idempotency token
-  -- This is so the frontend client doesn't submit duplicate jobs.
-  uuid_idempotency_token VARCHAR(36) NOT NULL,
-
-  -- ========== SUCCESS CASE ==========
-
-  -- If the job completes successfully, this is the result token.
-  -- This is only populated on a successful result.
-  on_success_downloaded_entity_token VARCHAR(32) DEFAULT NULL,
-
-  -- The type of the object will vary based on the type of the upload,
-  -- and we may include heuristics that auto-detect types in the future
-  on_success_downloaded_entity_type VARCHAR(32) DEFAULT NULL,
-
-  -- ========== UPLOAD DETAILS ==========
-
-  -- A "well defined" type of upload
-  -- The first upload type will be "hifigan_model"
-  download_type VARCHAR(32) NOT NULL,
-
-  -- Assume the same URL can be uploaded more than once (in case of errors or
-  -- people sharing models)
-
-  download_url VARCHAR(512) NOT NULL,
-
-  -- The user-supplied title of the upload
-  -- This might have different meanings for different upload types
-  -- The "name" of the voice model, which might be complicated.
-  -- If maybe_subject_token (etc.) is set, then it's authoritative instead.
-  title VARCHAR(255) NOT NULL,
-
-  -- ========== CREATOR DETAILS AND PREFERENCES ==========
-
-  -- Foreign key to user
-  creator_user_token VARCHAR(32) NOT NULL,
-
-  -- For abuse tracking.
-  -- Wide enough for IPv4/6
-  creator_ip_address VARCHAR(40) NOT NULL,
-
-  -- The creator can set a desired visibility for their data.
-  -- This does not always apply to every upload type.
-  -- Additionally, some upload types may require moderator approval prior
-  -- to being publicly listed, and this field has no bearing on that.
-  -- NB: DO NOT CHANGE THE ORDER OF THE ENUM VALUES DURING SCHEMA MIGRATIONS.
-  creator_set_visibility ENUM(
-    'public',
-    'hidden',
-    'private'
-  ) NOT NULL DEFAULT 'public',
-
-  -- ========== JOB SYSTEM DETAILS ==========
-
-  -- Jobs begin as "pending", then transition to other states.
-  --
-  --  * Pending = job is ready to go
-  --  * Started = job is running
-  --  * Complete_Success = job is done (success)
-  --  * Complete_Failure = job is done (failure)
-  --  * Attempt_Failed = job failed but may retry.
-  --  * Dead = job failed permanently.
-  --
-  -- Pending -> Started -> Complete_Success
-  --                    |-> Complete_Failure
-  --                    \-> Attempt_Failed -> Started -> { Complete, Failed, Dead }
-  status ENUM(
-    'pending',
-    'started',
-    'complete_success',
-    'complete_failure',
-    'attempt_failed',
-    'dead') NOT NULL DEFAULT 'pending',
-
-  -- We can track this against a "max_attempt_count"
-  attempt_count INT(3) NOT NULL DEFAULT 0,
-
-  -- If there is a failure, tell the user why.
-  failure_reason VARCHAR(512) DEFAULT NULL,
-
-  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-
-  -- Failed jobs will set a next attempt time.
-  -- Subsequent tries can increase the timeout.
-  -- Failures because of permissions require human intervention => [retry_at=null].
-  -- Failures because of invalid files are dead => [status=dead].
-  retry_at TIMESTAMP NULL,
-
-  -- INDICES --
-  PRIMARY KEY (id),
-  UNIQUE KEY (token),
-  UNIQUE KEY (uuid_idempotency_token),
-  KEY fk_on_download_type (download_type),
-  KEY fk_creator_user_token (creator_user_token),
-  KEY index_status (status),
-  KEY index_creator_ip_address (creator_ip_address)
-
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_migrations/2022-08-05-202110_vocoder_models/up.sql

@@ -1,128 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-CREATE TABLE vocoder_models (
-  -- Not used for anything except replication.
-  id BIGINT(20) NOT NULL AUTO_INCREMENT,
-
-  -- Effective "primary key" (PUBLIC)
-  token VARCHAR(32) NOT NULL,
-
-  -- The architecture of the vocoder.
-  -- Currently supported values: 'hifigan'
-  vocoder_type VARCHAR(32) NOT NULL,
-
-  -- ========== MUTABLE METADATA ==========
-
-  -- The "name" of the vocoder, which might be complicated.
-  title VARCHAR(255) NOT NULL,
-
-  -- The description of the model in markdown.
-  description_markdown TEXT NOT NULL,
-
-  -- Generated HTML (not user-editable).
-  description_rendered_html TEXT NOT NULL,
-
-  -- ========== MUTABLE MOD-ONLY METADATA ==========
-
-  -- Whether the vocoder is recommended for use
-  is_staff_recommended BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- ========== IMMUTABLE PROVENANCE AND METADATA ==========
-
-  -- Where the file was originally downloaded (if it was downloaded)
-  original_download_url VARCHAR(512) DEFAULT NULL,
-
-  -- The filename that was used at upload time (if available)
-  original_filename VARCHAR(255) DEFAULT NULL,
-
-  -- File characteristics
-  file_size_bytes INT(10) NOT NULL DEFAULT 0,
-
-  -- ========== CREATOR DETAILS ==========
-
-  -- The person that created the template.
-  creator_user_token VARCHAR(32) NOT NULL,
-
-  -- For abuse tracking.
-  -- Wide enough for IPv4/6
-  creator_ip_address_creation VARCHAR(40) NOT NULL,
-  creator_ip_address_last_update VARCHAR(40) NOT NULL,
-
-  -- (THIS MIGHT NOT BE USED)
-  -- NB: DO NOT SORT!
-  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
-  creator_set_visibility ENUM(
-    'public',
-    'hidden',
-    'private'
-  ) NOT NULL DEFAULT 'public',
-
-  -- ========== BUCKET STORAGE ==========
-
-  -- The pytorch model
-  -- For now, this will be a hash of the file contents.
-  -- NB: NOT UNIQUE! We can allow duplicate uploads.
-  private_bucket_hash CHAR(64) NOT NULL,
-
-  -- The "full url" of the bucket path, not including the hostname
-  private_bucket_object_name VARCHAR(255) NOT NULL,
-
-  -- ========== MODERATION DETAILS ==========
-
-  -- Mods have locked the ability for non-authors (ie. people who didn't upload this)
-  -- to assign the vocoder for use with their TTS models.
-  -- TTS models that already incorporate the vocoder will not be affected.
-  -- The vocoder will remain visible.
-  -- This is independent of other visibility controls (is_mod_disabled_from_author_use,
-  -- deleted_at, and creator_set_visibility)
-  is_mod_disabled_from_public_use BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- Mods have locked the ability for the author (ie. the person that uploaded this)
-  -- to assign the vocoder for use with their TTS models.
-  -- TTS models that already incorporate the vocoder will not be affected.
-  -- The vocoder will remain visible.
-  -- This is independent of other visibility controls (is_mod_disabled_from_public_use,
-  -- deleted_at, and creator_set_visibility)
-  is_mod_disabled_from_author_use BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- Extremely popular models may be locked from deletion or modification by users.
-  -- If true, only moderators can edit or delete the vocoder.
-  is_mod_author_editing_locked BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- If a moderator has comments.
-  maybe_mod_comments VARCHAR(255) DEFAULT NULL,
-
-  -- The last moderator that made changes.
-  maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
-
-  -- ========== VECTOR CLOCK ==========
-
-  -- Incremented with every update.
-  version INT NOT NULL DEFAULT 0,
-
-  -- ========== TIMESTAMPS ==========
-
-  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-
-  -- If this is removed by a mod.
-  -- It completely disappears from the system.
-  user_deleted_at TIMESTAMP NULL,
-  mod_deleted_at TIMESTAMP NULL,
-
-  -- INDICES --
-  PRIMARY KEY (id),
-  UNIQUE KEY (token),
-  KEY fk_creator_user_token (creator_user_token),
-  KEY fk_maybe_mod_user_token (maybe_mod_user_token),
-  KEY index_creator_ip_address_creation (creator_ip_address_creation),
-  KEY index_creator_ip_address_last_update (creator_ip_address_last_update),
-  KEY index_creator_set_visibility (creator_set_visibility),
-  KEY index_private_bucket_hash (private_bucket_hash),
-  KEY index_is_mod_disabled_from_public_use (is_mod_disabled_from_public_use),
-  KEY index_is_mod_disabled_from_author_use (is_mod_disabled_from_author_use),
-  KEY index_is_mod_author_editing_locked (is_mod_author_editing_locked)
-
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_migrations_squashed/tts_tables/tts_inference_jobs.sql

@@ -1,124 +0,0 @@
--- NB: This is a manually squashed view of all the CREATE and ALTER statements,
--- with comments attached to the fields for centralized documentation.
-
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-CREATE TABLE tts_inference_jobs (
-  id BIGINT(20) NOT NULL AUTO_INCREMENT,
-
-  -- Effective "primary key" (PUBLIC)
-  -- This is so the in-progress results can be looked up by the UI.
-  token VARCHAR(32) NOT NULL,
-
-  -- Idempotency token from client
-  -- This is so the frontend client doesn't submit duplicate jobs.
-  uuid_idempotency_token VARCHAR(36) NOT NULL,
-
-  -- ========== SUCCESS CASE ==========
-
-  -- If the job completes successfully, this is the result token.
-  -- This is only populated on a successful result.
-  on_success_result_token VARCHAR(32) DEFAULT NULL,
-
-  -- ========== INFERENCE DETAILS ==========
-
-  -- The model to use.
-  -- This also determines which architecture we're using.
-  model_token VARCHAR(32) NOT NULL,
-
-  -- If set, use a different pretrained vocoder.
-  maybe_override_pretrained_vocoder VARCHAR(64) DEFAULT NULL,
-
-  -- The raw, unprocessed user input.
-  raw_inference_text TEXT NOT NULL,
-
-  -- ========== CREATOR DETAILS ==========
-
-  -- Foreign key to user
-  -- If no user is logged in, this is null.
-  maybe_creator_user_token VARCHAR(32) DEFAULT NULL,
-
-  -- Based on a cookie sent by the frontend.
-  -- We'll save this even if the user is logged in.
-  maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL,
-
-  -- For abuse tracking.
-  -- Wide enough for IPv4/6
-  creator_ip_address VARCHAR(40) NOT NULL,
-
-  -- (THIS MIGHT NOT BE USED)
-  -- NB: DO NOT SORT!
-  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
-  creator_set_visibility ENUM(
-    'public',
-    'hidden',
-    'private'
-  ) NOT NULL DEFAULT 'public',
-
-  -- ========== SOURCE METADATA ==========
-
-  is_from_api BOOLEAN NOT NULL DEFAULT FALSE,
-  is_for_twitch BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- If true, the request gets routed to a special "debug" worker.
-  is_debug_request BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- ========== JOB SYSTEM DETAILS ==========
-
-  -- Jobs begin as "pending", then transition to other states.
-  --
-  --  * Pending = job is ready to go
-  --  * Started = job is running
-  --  * Complete_Success = job is done (success)
-  --  * Complete_Failure = job is done (failure)
-  --  * Attempt_Failed = job failed but may retry.
-  --  * Dead = job failed permanently.
-  --
-  -- Pending -> Started -> Complete_Success
-  --                    |-> Complete_Failure
-  --                    \-> Attempt_Failed -> Started -> { Complete, Failed, Dead }
-  status ENUM(
-      'pending',
-      'started',
-      'complete_success',
-      'complete_failure',
-      'attempt_failed',
-      'dead') NOT NULL DEFAULT 'pending',
-
-  -- Priority *increases*, so a level of 2 will be higher than 1.
-  -- These are the level semantics currently:
-  --   - All jobs from anonymous FakeYou users have level 0.
-  --   - All jobs from logged in FakeYou users have level 1.
-  --   - All jobs from Twitch TTS (unpaid) have level 10 (ten).
-  --   - (There will be future levels for paid Twitch and social FakeYou rewards.)
-  priority_level TINYINT UNSIGNED NOT NULL DEFAULT 0,
-
-  -- We can track this against a "max_attempt_count"
-  attempt_count INT(3) NOT NULL DEFAULT 0,
-
-  -- If there is a failure, tell the user why.
-  failure_reason VARCHAR(512) DEFAULT NULL,
-
-  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-
-  -- Failed jobs will set a next attempt time.
-  -- Subsequent tries can increase the timeout.
-  -- Failures because of permissions require human intervention => [retry_at=null].
-  -- Failures because of invalid files are dead => [status=dead].
-  retry_at TIMESTAMP NULL,
-
-  -- INDICES --
-  PRIMARY KEY (id),
-  UNIQUE KEY (token),
-  UNIQUE KEY (uuid_idempotency_token),
-  KEY fk_on_success_result_token (on_success_result_token),
-  KEY fk_model_token (model_token),
-  KEY fk_maybe_creator_user_token (maybe_creator_user_token),
-  KEY index_status (status),
-  KEY index_creator_ip_address (creator_ip_address),
-  KEY index_is_debug_request (is_debug_request)
-
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_migrations_squashed/tts_tables/tts_models.sql

@@ -1,217 +0,0 @@
--- NB: This is a manually squashed view of all the CREATE and ALTER statements,
--- with comments attached to the fields for centralized documentation.
-
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-CREATE TABLE tts_models (
-  -- Not used for anything except replication.
-  id BIGINT(20) NOT NULL AUTO_INCREMENT,
-
-  -- Effective "primary key" (PUBLIC)
-  token VARCHAR(32) NOT NULL,
-
-  -- NB: DO NOT CHANGE ORDER; APPEND ONLY!
-  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
-  tts_model_type ENUM(
-      'not-set',
-      'tacotron2',
-      'glowtts',
-      'glowtts-vocodes',
-      'talknet'
-  ) NOT NULL DEFAULT 'not-set',
-
-  -- The name of the type of pipeline to use, eg "legacy_fakeyou", "legacy_vocodes", "english_v1", etc.
-  -- New pipelines may be added at any point in the future and the source of truth for them will mostly
-  -- live in the `storyteller-ml` repository.
-  -- Older models and brand new models will have this field set to null, and we'll infer which value to
-  -- lazily backfill based on the creation date and language setting.
-  text_pipeline_type VARCHAR(64) DEFAULT NULL,
-
-  -- For models distributed as archives/bundles (eg. TalkNet), we can pack
-  -- in a vocoder. Note that this vocoder does not have to be used.
-  has_self_contained_vocoder BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- For models distributed as archives/bundles (eg. TalkNet), we can pack
-  -- in a duration model.
-  has_self_contained_duration_model BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- For models distributed as archives/bundles (eg. TalkNet), we can pack
-  -- in a pitch model.
-  has_self_contained_pitch_model BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- NB(2022-07-05): THIS IS NOW MEANINGLESS AND SUPERSEDED BY `TEXT_PIPELINE_TYPE`.
-  -- NB: DO NOT CHANGE ORDER; APPEND ONLY!
-  -- How text should be handled.
-  text_preprocessing_algorithm ENUM(
-      'basic',
-      'arpabet'
-  ) NOT NULL DEFAULT 'basic',
-
-  -- If set, use a different pretrained vocoder.
-  -- If not set, use the website default. (Currently HifiGan-SS)
-  maybe_default_pretrained_vocoder VARCHAR(64) DEFAULT NULL,
-
-  -- If set, we'll use a custom vocoder (from table vocoder_models) instead of a default vocoder.
-  -- If not set, we'll use `maybe_default_pretrained_vocoder` (or the website default).
-  maybe_custom_vocoder_token VARCHAR(32) DEFAULT NULL,
-
-  -- Optional Pointer to a newer version of the voice
-  -- If there's a newer version, we can disable this one.
-  -- maybe_updated_model_token VARCHAR(32) DEFAULT NULL,
-
-  -- We can set as a combination of ['username' + 'voice-name']
-  -- There can be public aliases for voices, eg. a voice's default model.
-  -- A user can change this.
-  -- As such, these should not be foreign keys.
-  -- updatable_slug VARCHAR(64) NOT NULL,
-
-  -- Can be linked to a well-known voice
-  -- maybe_subject_token VARCHAR(32) DEFAULT NULL,
-  -- maybe_actor_subject_token VARCHAR(32) DEFAULT NULL,
-
-  -- ========== UNSTRUCTURED METADATA ==========
-
-  -- The full IETF BCP47 language tag (eg. en, en-US, es-419, ja-JP, pt, etc.)
-  ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en',
-
-  -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es")
-  ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en',
-
-  -- The "name" of the voice model, which might be complicated.
-  -- If maybe_subject_token (etc.) is set, then it's authoritative instead.
-  title VARCHAR(255) NOT NULL,
-
-  -- The description of the model in markdown.
-  description_markdown TEXT NOT NULL,
-
-  -- Generated HTML (not user-editable).
-  description_rendered_html TEXT NOT NULL,
-
-  -- eg. Bill Clinton, Batman, or Goku.
-  voice_name VARCHAR(255) DEFAULT NULL,
-
-  -- Optional; eg. Sean Schemmel
-  actor_name VARCHAR(255) DEFAULT NULL,
-
-  -- Optional; eg. if the voice is "happy" or a singer "a-capella", etc.
-  characteristic VARCHAR(255) DEFAULT NULL,
-
-  -- Optional; eg. if you train the model more than once.
-  version_string VARCHAR(255) DEFAULT NULL,
-
-  -- Whether the voice features on the front of FakeYou or Storyteller Twitch TTS
-  is_front_page_featured BOOLEAN NOT NULL DEFAULT FALSE,
-  is_twitch_featured BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- A suggested bot command prefix, eg. 'sonic' that could be interpreted as '/sonic'
-  -- by a TTS system. These are unique strings that can only be applied to one model at
-  -- a time at most. Only moderators can set this, and it's only for the best voices.
-  maybe_suggested_unique_bot_command VARCHAR(255) DEFAULT NULL,
-
-  -- ========== IMMUTABLE PROVENANCE AND METADATA ==========
-
-  -- Where the file was originally downloaded (if it was downloaded)
-  original_download_url VARCHAR(512) DEFAULT NULL,
-
-  -- The filename that was used at upload time (if available)
-  original_filename VARCHAR(255) DEFAULT NULL,
-
-  -- File characteristics
-  file_size_bytes INT(10) NOT NULL DEFAULT 0,
-
-  -- ========== CREATOR DETAILS ==========
-
-  -- The person that created the template.
-  creator_user_token VARCHAR(32) NOT NULL,
-
-  -- For abuse tracking.
-  -- Wide enough for IPv4/6
-  creator_ip_address_creation VARCHAR(40) NOT NULL,
-  creator_ip_address_last_update VARCHAR(40) NOT NULL,
-
-  -- (THIS MIGHT NOT BE USED)
-  -- NB: DO NOT SORT!
-  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
-  creator_set_visibility ENUM(
-    'public',
-    'hidden',
-    'private'
-   ) NOT NULL DEFAULT 'public',
-
-  -- ========== BUCKET STORAGE ==========
-
-  -- The pytorch model
-  -- For now, this will be a hash of the file contents.
-  -- NB: NOT UNIQUE! We can allow duplicate uploads.
-  private_bucket_hash CHAR(64) NOT NULL,
-  -- The "full url" version of the path
-  private_bucket_object_name VARCHAR(255) NOT NULL,
-
-  -- Whether the file is a zip archive (rather than a model file).
-  private_bucket_object_is_archive BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- ========== RATING AND RANKING ==========
-
-  -- Calculated average, on a scale of 0-100
-  -- Null with zero ratings.
-  -- calculated_average_score INT(3) DEFAULT NULL,
-
-  -- Other metrics
-  -- calculated_total_ratings_submitted_count INT(10) NOT NULL DEFAULT 0,
-  -- calculated_total_uses_count BIGINT(10) NOT NULL DEFAULT 0,
-
-  -- ========== MODERATION DETAILS ==========
-
-  -- Mods have to approve of tts models for them to show up in a public index.
-  -- Models can be used by the author (or other parties that know the URL) before
-  -- they're approved, but unapproved models won't show up in public indices.
-  -- This is independent of other visibility controls (is_mod_disabled, deleted_at,
-  -- and creator_set_visibility)
-  is_public_listing_approved BOOLEAN DEFAULT NULL,
-
-  -- Extremely popular models may be locked from deletion or modification by users.
-  is_locked_from_user_modification BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- In this case, a moderator disables it from being used for inference.
-  -- This also disables it for the creator.
-  -- Unlike deletion, it remains "visible" to those that have access.
-  is_locked_from_use BOOLEAN NOT NULL DEFAULT FALSE,
-
-  -- If a moderator has comments.
-  maybe_mod_comments VARCHAR(255) DEFAULT NULL,
-
-  -- The last moderator that made changes.
-  maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
-
-  -- ========== VECTOR CLOCK ==========
-
-  -- Incremented with every update.
-  version INT NOT NULL DEFAULT 0,
-
-  -- ========== TIMESTAMPS ==========
-
-  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-
-  -- If this is removed by a mod.
-  -- It completely disappears from the system.
-  user_deleted_at TIMESTAMP NULL,
-  mod_deleted_at TIMESTAMP NULL,
-
-  -- INDICES --
-  PRIMARY KEY (id),
-  UNIQUE KEY (token),
-  KEY fk_creator_user_token (creator_user_token),
-  KEY fk_maybe_mod_user_token (maybe_mod_user_token),
-  KEY fk_maybe_custom_vocoder_token (maybe_custom_vocoder_token),
-  KEY index_creator_ip_address_creation (creator_ip_address_creation),
-  KEY index_creator_ip_address_last_update (creator_ip_address_last_update),
-  KEY index_creator_set_visibility (creator_set_visibility),
-  KEY index_private_bucket_hash (private_bucket_hash),
-  KEY index_is_public_listing_approved (is_public_listing_approved),
-  KEY index_is_locked_from_user_modification (is_locked_from_user_modification),
-  KEY index_is_locked_from_use (is_locked_from_use)
-
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_sql/analytics/tts/tts_category_assignments.sql

@@ -0,0 +1,43 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Select TTS models within a category
+-- (NOTE CATEGORIES ARE *SOFT DELETED* AND THIS MAY MIS-ATTRIBUTE)
+select
+    tts.token,
+    tts.ietf_language_tag,
+    tts.title
+from tts_category_assignments as cat
+join tts_models as tts
+on tts.token = cat.model_token
+where
+    cat.category_token = 'CAT:n06k9gfgq20';
+
+-- Select TTS models within a category
+select
+    tts.token,
+    tts.ietf_language_tag,
+    tts.title
+from tts_category_assignments as cat
+         join tts_models as tts
+              on tts.token = cat.model_token
+where
+        cat.category_token = 'CAT:jtdwpy6ptrn'
+AND cat.deleted_at IS NULL;
+
+
+-- Select TTS models within a category
+-- Exclude given languages.
+select
+    tts.token,
+    tts.ietf_language_tag,
+    tts.title
+from tts_category_assignments as cat
+         join tts_models as tts
+              on tts.token = cat.model_token
+where
+        cat.category_token = 'CAT:jtdwpy6ptrn'
+  AND tts.ietf_language_tag NOT IN ('es', 'es-419', 'es-ES', 'es-MX')
+  AND cat.deleted_at IS NULL;
+

_sql/analytics/tts/tts_inference_jobs.sql

@@ -0,0 +1,82 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Kill pending jobs
+update tts_inference_jobs set status = 'dead' where status = 'pending';
+
+-- Total pending jobs
+select count(*)
+from tts_inference_jobs
+where status = 'pending';
+
+-- Total pending API jobs
+select count(*)
+from tts_inference_jobs
+where status = 'pending'
+and is_from_api = true;
+
+-- Top IP addresses making requests
+select distinct creator_ip_address, count(*) as attempts
+from tts_inference_jobs
+where status = 'pending'
+group by creator_ip_address
+order by attempts desc
+limit 50;
+
+-- Top voices in requests
+select distinct model_token, count(*) as attempts
+from tts_inference_jobs
+where status = 'pending'
+group by model_token
+order by attempts desc
+    limit 50;
+
+
+-- Kill pending jobs
+update tts_inference_jobs
+set status = 'dead'
+where status = 'pending';
+
+-- Kill all waiting and in-progress jobs
+update tts_inference_jobs
+set status = 'dead'
+where status IN ('pending', 'started', 'attempt_failed');
+
+-- Sample the pending inference text
+select creator_ip_address,
+       model_token,
+       maybe_creator_user_token,
+       raw_inference_text
+from tts_inference_jobs
+where status = 'pending';
+
+-- Sample the pending inference text (truncated)
+select creator_ip_address,
+       model_token,
+       maybe_creator_user_token,
+       TRIM(REPLACE(SUBSTRING(raw_inference_text, 1, 50), '\n', ''))
+from tts_inference_jobs
+where status = 'pending'
+limit 100;
+
+-- Sample long pending texts
+select creator_ip_address,
+       model_token,
+       maybe_creator_user_token,
+       raw_inference_text
+from tts_inference_jobs
+where status = 'pending'
+and length(raw_inference_text) > 100
+limit 100;
+
+-- Delete old TTS inference jobs (1)
+DELETE FROM tts_inference_jobs
+WHERE status IN ('dead', 'complete_success', 'complete_failure')
+LIMIT 100000;
+
+-- Delete old TTS inference jobs (2)
+-- Roughly 20 seconds to delete 100k, 4 minutes for 1M.
+DELETE FROM tts_inference_jobs
+WHERE created_at < ( CURDATE() - INTERVAL 1 DAY )
+LIMIT 100000;

_sql/analytics/tts/tts_models.sql

@@ -0,0 +1,302 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+--
+-- Top 100 models all time by use count
+--
+select
+    m.token,
+    m.text_pipeline_type,
+    m.ietf_language_tag,
+    m.title,
+    u.username,
+    r.use_count,
+    m.user_deleted_at,
+    m.mod_deleted_at,
+    m.created_at
+from (
+         select model_token, count(*) as use_count
+         from tts_results
+         group by model_token
+     ) as r
+         join tts_models as m
+              on m.token = r.model_token
+         join users as u
+              on u.token = m.creator_user_token
+order by r.use_count desc
+    limit 100;
+
+--
+-- Same, but simpler...
+--
+select m.token, m.title, r.use_count from (
+  select model_token, count(*) as use_count from tts_results
+  group by model_token
+  order by use_count desc limit 100
+) as r
+  join tts_models as m
+  on m.token = r.model_token;
+
+--
+-- Top 100 models by use count, last 5 days
+-- Limited due to tmux scrollback.
+--
+select
+    m.token,
+    m.text_pipeline_type,
+    m.ietf_language_tag,
+    m.title,
+    u.username,
+    r.use_count,
+    m.created_at,
+    m.user_deleted_at,
+    m.mod_deleted_at
+from (
+    select model_token, count(*) as use_count
+    from tts_results
+    where created_at > ( CURDATE() - INTERVAL 12 HOUR )
+    group by model_token
+) as r
+    join tts_models as m
+    on m.token = r.model_token
+    join users as u
+    on u.token = m.creator_user_token
+order by r.use_count desc
+    limit 25;
+
+--
+-- Top 20 ***NEW*** models to share in #announcements
+-- Can be scoped by creator.
+--
+select
+    m.token,
+    m.text_pipeline_type,
+    m.ietf_language_tag,
+    m.title,
+    u.username,
+    r.use_count,
+    m.user_deleted_at,
+    m.mod_deleted_at,
+    m.created_at
+from (
+         select model_token, count(*) as use_count
+         from tts_results
+         where created_at > ( CURDATE() - INTERVAL 5 DAY )
+         group by model_token
+     ) as r
+         join tts_models as m
+              on m.token = r.model_token
+         join users as u
+              on u.token = m.creator_user_token
+where m.created_at > ( CURDATE() - INTERVAL 31 DAY )
+  AND u.username IN ( 'vegito1089', 'justinjohn0306')
+order by r.use_count desc
+    limit 100;
+
+--
+-- Top 100 models by use count, last 5 days, not spanish, etc.
+-- Limited due to tmux scrollback.
+--
+select
+    m.token,
+    m.text_pipeline_type,
+    m.ietf_language_tag,
+    m.title,
+    u.username,
+    r.use_count,
+    m.user_deleted_at,
+    m.mod_deleted_at,
+    m.created_at
+from (
+    select model_token, count(*) as use_count
+    from tts_results
+    where created_at > ( CURDATE() - INTERVAL 5 DAY )
+    group by model_token
+) as r
+    join tts_models as m
+    on m.token = r.model_token
+    join users as u
+    on u.token = m.creator_user_token
+    where m.ietf_language_tag NOT IN ('es', 'es-419', 'es-ES', 'es-MX', 'pt-BR')
+order by r.use_count desc
+    limit 100;
+
+--
+-- Top 100 models by use count, last 5 days, in specific language.
+-- Limited due to tmux scrollback.
+--
+-- English: ('en', 'en-US', 'en-AU', 'en-CA', 'en-GB')
+-- Spanish: ('es', 'es-419', 'es-AR', 'es-CL', 'es-CO', 'es-ES', 'es-MX', 'es-PE', 'es-US')
+-- Italian: ('it', 'it-CH', 'it-IT')
+-- Portuguese: ('pt', 'pt-BR')
+--
+select
+    m.token,
+    m.title,
+    m.text_pipeline_type,
+    m.ietf_language_tag,
+    u.username,
+    r.use_count,
+    m.user_deleted_at,
+    m.mod_deleted_at,
+    m.created_at
+from (
+         select model_token, count(*) as use_count
+         from tts_results
+         where created_at > ( CURDATE() - INTERVAL 30 DAY )
+         group by model_token
+     ) as r
+         join tts_models as m
+              on m.token = r.model_token
+         join users as u
+              on u.token = m.creator_user_token
+where m.ietf_language_tag IN ('en', 'en-US', 'en-AU', 'en-CA', 'en-GB')
+order by r.use_count desc
+    limit 100;
+
+--
+-- Most popular voices by use count over 5-day window, single language, single user.
+--
+select
+    m.token,
+    m.title,
+    m.ietf_language_tag,
+    u.username,
+    r.use_count,
+    m.user_deleted_at,
+    m.mod_deleted_at,
+    m.created_at
+from (
+    select model_token, count(*) as use_count
+    from tts_results
+    where created_at > ( CURDATE() - INTERVAL 5 DAY )
+    group by model_token
+) as r
+    join tts_models as m
+    on m.token = r.model_token
+    join users as u
+    on u.token = m.creator_user_token
+    where
+        m.ietf_language_tag NOT IN ('es', 'es-419', 'es-ES', 'es-MX', 'pt-BR')
+        AND u.username = 'vegito1089'
+order by r.use_count desc
+    limit 100;
+
+--
+-- Most popular deleted models
+--
+select
+    m.token,
+    m.title,
+    m.ietf_language_tag,
+    u.username,
+    r.use_count,
+    m.user_deleted_at,
+    m.mod_deleted_at,
+    m.created_at
+from (
+    select model_token, count(*) as use_count
+    from tts_results
+    where model_token IN (
+        select token
+        from tts_models
+        where
+            user_deleted_at IS NOT NULL
+            OR mod_deleted_at IS NOT NULL
+    )
+    group by model_token
+) as r
+    join tts_models as m
+    on m.token = r.model_token
+    join users as u
+    on u.token = m.creator_user_token
+order by r.use_count desc
+    limit 100;
+
+--
+-- Most popular deleted models (only by users, not mods)
+-- Mod-deleted models were probably on purpose
+--
+select
+    m.token,
+    m.title,
+    m.ietf_language_tag,
+    u.username,
+    r.use_count,
+    m.user_deleted_at,
+    m.created_at
+from (
+         select model_token, count(*) as use_count
+         from tts_results
+         where model_token IN (
+             select token
+             from tts_models
+             where
+                 user_deleted_at IS NOT NULL
+         )
+         group by model_token
+     ) as r
+         join tts_models as m
+              on m.token = r.model_token
+         join users as u
+              on u.token = m.creator_user_token
+order by r.use_count desc
+    limit 100;
+
+
+--
+-- Models uploaded recently (to check if they use the right text_pipeline_type,
+-- vocoder -todo-, etc.)
+--
+select
+    m.token,
+    m.title,
+    m.text_pipeline_type,
+    m.ietf_language_tag,
+    u.username,
+    r.use_count,
+    m.user_deleted_at,
+    m.mod_deleted_at,
+    m.created_at
+from (
+         select model_token, count(*) as use_count
+         from tts_results
+         where created_at > ( CURDATE() - INTERVAL 5 DAY )
+         group by model_token
+     ) as r
+         join tts_models as m
+              on m.token = r.model_token
+         join users as u
+              on u.token = m.creator_user_token
+order by m.created_at desc
+    limit 100;
+
+--
+-- Models deleted recently (voice actor take down)
+--
+select
+    m.token,
+    m.title,
+    m.text_pipeline_type,
+    m.ietf_language_tag,
+    u.username,
+    r.use_count,
+    m.created_at,
+    m.user_deleted_at,
+    m.mod_deleted_at
+from (
+         select model_token, count(*) as use_count
+         from tts_results
+         where created_at > ( CURDATE() - INTERVAL 30 DAY )
+         group by model_token
+     ) as r
+         join tts_models as m
+              on m.token = r.model_token
+         join users as u
+              on u.token = m.creator_user_token
+where m.mod_deleted_at > ( CURDATE() - INTERVAL 14 DAY )
+   or m.user_deleted_at > ( CURDATE() - INTERVAL 14 DAY )
+order by r.use_count desc
+    limit 500;
+

_sql/analytics/tts/tts_results.sql

@@ -0,0 +1,101 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+--
+-- Usage count in the past 24 hours
+--
+select count(*) as use_count
+from tts_results
+where created_at > ( CURDATE() - INTERVAL 1 DAY )
+
+--
+-- Usage count in the past week
+--
+select count(*) as use_count
+from tts_results
+where created_at > ( CURDATE() - INTERVAL 7 DAY )
+
+-- On-prem worker mix
+select count(*) as use_count
+from tts_results
+where created_at > ( CURDATE() - INTERVAL 1 MINUTE )
+and is_generated_on_prem IS TRUE
+
+-- Calculate a percentage of on-prem worker capacity
+select count(*) as on_prem_count
+from (
+  select is_generated_on_prem
+  from tts_results
+  order by id desc limit 1000
+) as sample
+where sample.is_generated_on_prem IS TRUE;
+
+
+-- Find TTS results for a single model
+select count(*) from tts_results
+where model_token = 'TM:ztt5s1be5tq6';
+
+-- Find TTS results for a single model that are not deleted
+select count(*) from tts_results
+where model_token = 'TM:ztt5s1be5tq6'
+and mod_deleted_at is NULL;
+
+-- Sample results for a single model
+select
+    TRIM(REPLACE(SUBSTRING(raw_inference_text, 1, 50), '\n', ''))
+from tts_results
+where model_token='TM:yt4gfbkngsjj'
+order by id desc
+limit 150;
+
+-- Search for results by eminem
+select
+    token,
+    created_at,
+    raw_inference_text
+from  tts_results
+where
+    model_token IN ('TM:27fj0gsh11pd', 'TM:8h9bfjgabeer', 'TM:pdf9c1anbdjq')
+    AND raw_inference_text LIKE "%rave%"
+  AND raw_inference_text LIKE "%sound%"
+  AND created_at > ( CURDATE() - INTERVAL 5 MONTH )
+ORDER BY created_at DESC
+
+
+
+-- Delete TTS results for a single model
+-- Do this in short batches so a lock isn't held for prohibitively long.
+update tts_results
+set mod_deleted_at = NOW()
+where model_token = 'TM:ztt5s1be5tq6'
+and mod_deleted_at IS NULL
+limit 5000;
+
+--- Search for particular text by a particular model
+SELECT
+    raw_inference_text,
+    maybe_creator_user_token,
+    created_at
+FROM
+    tts_results
+WHERE
+    model_token = 'TM:ztt5s1be5tq6'
+    AND raw_inference_text LIKE '%fakeyou%'
+
+---
+-- WTF WHY IS THIS SLOW???
+SELECT
+    res.id,
+    model.title,
+    res.raw_inference_text,
+    res.maybe_creator_user_token
+FROM
+    tts_results AS res
+JOIN
+    tts_models as model
+ON
+    res.model_token = model.token
+ORDER BY res.id
+LIMIT 10;
+

_sql/analytics/users/paying_user_interests.sql

@@ -0,0 +1,119 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- TODO: Do an analysis of voice binned by # of paying customers that use it.
+
+-- The most popular models amongst paying subscribers.
+-- This should give us insight as to which models subscribers pay us for.
+SELECT *
+FROM (
+         SELECT
+             t.token,
+             t.title,
+             count(*) as use_count,
+             t.text_pipeline_type,
+             t.ietf_language_tag,
+             t.created_at,
+             t.user_deleted_at,
+             t.mod_deleted_at
+         FROM
+             tts_inference_jobs as j
+                 JOIN
+             tts_models as t
+             ON
+                     t.token = j.model_token
+         WHERE j.maybe_creator_user_token IN
+               (
+                   select distinct user_token
+                   from user_subscriptions
+               )
+           AND
+                 j.created_at > ( CURDATE() - INTERVAL 1 DAY )
+         GROUP BY t.token
+         ORDER BY use_count DESC
+     ) as x
+     WHERE
+        x.use_count > 10;
+
+
+-- The first voice a paying customer uses
+-- TODO: List *all* of the voices a paying customer uses before they subscribe, not just the first.
+-- select first row of partially aggregate query: https://stackoverflow.com/a/73157541
+SELECT *
+FROM (
+         SELECT
+             t.token,
+             t.title,
+             count(*) as use_count,
+             t.text_pipeline_type,
+             t.ietf_language_tag,
+             t.created_at,
+             t.user_deleted_at,
+             t.mod_deleted_at
+         FROM tts_models AS t
+                  JOIN
+              (
+                  SELECT
+                      model_token
+                  FROM (
+                           SELECT
+                               model_token,
+                               maybe_creator_user_token,
+                               row_number() over(partition by maybe_creator_user_token order by id asc) as rownum
+                           FROM
+                               tts_inference_jobs
+                           WHERE maybe_creator_user_token IN
+                                 (
+                                     select distinct user_token
+                                     from user_subscriptions
+                                     where created_at > (CURDATE() - INTERVAL 2 DAY)
+                                 )
+                       ) AS x
+                  WHERE rownum = 1
+                  ORDER BY model_token desc
+              ) AS y
+              ON t.token = y.model_token
+         GROUP BY t.token
+     ) AS z
+ORDER BY use_count desc
+
+-- The last voice a paying customer uses
+-- select first row of partially aggregate query: https://stackoverflow.com/a/73157541
+SELECT *
+FROM (
+    SELECT
+      t.token,
+      t.title,
+      count(*) as use_count,
+      t.text_pipeline_type,
+      t.ietf_language_tag,
+      t.created_at,
+      t.user_deleted_at,
+      t.mod_deleted_at
+    FROM tts_models AS t
+    JOIN
+        (
+            SELECT
+                model_token
+            FROM (
+                SELECT
+                    model_token,
+                    maybe_creator_user_token,
+                    row_number() over(partition by maybe_creator_user_token order by id desc) as rownum
+                FROM
+                    tts_inference_jobs
+                WHERE maybe_creator_user_token IN
+                      (
+                          select distinct user_token
+                          from user_subscriptions
+                      )
+            ) AS x
+            WHERE rownum = 1
+            ORDER BY model_token desc
+          ) AS y
+    ON t.token = y.model_token
+    GROUP BY t.token
+) AS z
+ORDER BY use_count desc
+

_sql/analytics/users/paying_user_retention.sql

@@ -0,0 +1,124 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+
+-- Number of paid users (inaccurate)
+select count(*) from users where maybe_stripe_customer_id IS NOT NULL;
+
+-- Number of paid users (inaccurate)
+select count(*) from user_subscriptions;
+
+-- Number of paid users
+-- NB: "active" status users are current, treating "past_due" as active too (for now)
+select count(*) from user_subscriptions where maybe_stripe_subscription_status NOT IN ("canceled", "incomplete", "incomplete_expired");
+
+-- ==================== TEXT TO SPEECH====================
+
+-- All paid subscribers generating results in the last week
+select u.username, count(*) as use_count
+from users as u
+join tts_inference_jobs as j
+on u.token = j.maybe_creator_user_token
+WHERE u.token IN (
+    select distinct user_token
+    from user_subscriptions
+    where maybe_stripe_subscription_status NOT IN ("canceled", "incomplete", "incomplete_expired")
+)
+AND j.created_at > NOW() - interval 7 day
+GROUP BY u.username
+ORDER BY use_count desc
+
+-- Paid subscribers generating results in the last month (with # threshold)
+select username, use_count
+FROM (
+    select u.username, count(*) as use_count
+    from users as u
+             join tts_inference_jobs as j
+                  on u.token = j.maybe_creator_user_token
+    WHERE u.token IN (
+        select distinct user_token
+        from user_subscriptions
+        where maybe_stripe_subscription_status NOT IN ("canceled", "incomplete", "incomplete_expired")
+    )
+    AND j.created_at > NOW() - interval 31 day
+    GROUP BY u.username
+    ORDER BY use_count desc
+) as t
+WHERE use_count > 10
+
+-- Paid users generating results in the last month
+select u.username, count(*) as use_count
+from users as u
+      join tts_inference_jobs as j
+           on u.token = j.maybe_creator_user_token
+WHERE u.token IN (
+     select distinct user_token
+     from user_subscriptions
+     where maybe_stripe_subscription_status NOT IN ("canceled", "incomplete", "incomplete_expired")
+)
+AND j.created_at > NOW() - interval 1 day
+GROUP BY u.username
+ORDER BY use_count desc
+
+-- ==================== WAV2LIP ====================
+
+select u.username, count(*) as use_count
+from users as u
+         join w2l_inference_jobs as j
+              on u.token = j.maybe_creator_user_token
+WHERE u.token IN (
+    select distinct user_token
+    from user_subscriptions
+    where maybe_stripe_subscription_status NOT IN ("canceled", "incomplete", "incomplete_expired")
+)
+  AND j.created_at > NOW() - interval 31 day
+GROUP BY u.username
+ORDER BY use_count desc
+
+select username, use_count
+FROM (
+         select u.username, count(*) as use_count
+         from users as u
+                  join w2l_inference_jobs as j
+                       on u.token = j.maybe_creator_user_token
+         WHERE u.token IN (
+             select distinct user_token
+             from user_subscriptions
+             where maybe_stripe_subscription_status NOT IN ("canceled", "incomplete", "incomplete_expired")
+         )
+           AND j.created_at > NOW() - interval 31 day
+         GROUP BY u.username
+         ORDER BY use_count desc
+     ) as t
+WHERE use_count >=5;
+
+-- ==================== TTS MODELS ====================
+
+select u.username, count(*) as use_count
+from users as u
+         join tts_model_upload_jobs as j
+              on u.token = j.creator_user_token
+WHERE u.token IN (
+    select distinct user_token
+    from user_subscriptions
+    where maybe_stripe_subscription_status NOT IN ("canceled", "incomplete", "incomplete_expired")
+)
+  AND j.created_at > NOW() - interval 1 day
+GROUP BY u.username
+ORDER BY use_count desc
+
+-- ==================== W2L TEMPLATES ====================
+
+select u.username, count(*) as use_count
+from users as u
+         join w2l_template_upload_jobs as j
+              on u.token = j.creator_user_token
+WHERE u.token IN (
+    select distinct user_token
+    from user_subscriptions
+    where maybe_stripe_subscription_status NOT IN ("canceled", "incomplete", "incomplete_expired")
+)
+  AND j.created_at > NOW() - interval 1 day
+GROUP BY u.username
+ORDER BY use_count desc

_sql/analytics/users/stripe_sigma_queries.sql

@@ -0,0 +1,30 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- NB: These are for use in Stripe Sigma, not MySQL
+-- NB: You must use single quotes instead of double quotes when comparing string values.
+
+-- Histogram of customers by country
+select address_country, count(*)
+from customers
+group by address_country
+
+select customer_id, subscriptions.status
+from subscriptions
+
+
+select distinct customer_id
+from subscriptions
+where subscriptions.status = 'active'
+or subscriptions.status = 'past_due'
+
+select address_country, count(*) as country_count
+from customers
+where id IN (
+    select distinct customer_id
+    from subscriptions
+    where subscriptions.status = 'active'
+    or subscriptions.status = 'past_due'
+)
+group by address_country

_sql/analytics/users/user_top_contributors.sql

@@ -0,0 +1,44 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Top TTS model uploaders
+select
+    u.id,
+    u.token,
+    u.username,
+    u.created_at,
+    u.maybe_stripe_customer_id,
+    u.maybe_loyalty_program_key,
+    m.upload_count
+from users as u
+join
+(
+  select
+      count(*) as upload_count,
+      creator_user_token
+  from tts_models
+  group by creator_user_token
+  order by upload_count desc
+) as m
+on u.token = m.creator_user_token
+where m.upload_count >= 5;
+
+
+
+-- Upgrade top TTS model uploaders to a loyalty plan
+update users
+set maybe_loyalty_program_key = 'fakeyou_contributor'
+where token IN (
+  select creator_user_token from
+    (
+      select count(*) as upload_count,
+      creator_user_token
+      from tts_models
+      group by creator_user_token
+      having upload_count >= 10
+      order by upload_count desc
+  ) as t
+);
+
+

_sql/analytics/w2l/w2l_inference_jobs.sql

@@ -0,0 +1,7 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Pending job count
+select count(*) from w2l_inference_jobs where status='pending';
+

_sql/analytics/w2l/w2l_template_upload_jobs.sql

@@ -0,0 +1,28 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Get a count of pending jobs
+select count(*) from w2l_template_upload_jobs where status='pending';
+
+-- See pending download URLs:
+select download_url from w2l_template_upload_jobs where status IN ('pending', 'started', 'attempt_failed');
+
+-- See old jobs that might be stuck as "started"
+select count(*)
+from w2l_template_upload_jobs
+where status='started'
+and created_at < ( CURDATE() - INTERVAL 1 DAY );
+
+-- Unstick old jobs that might be stuck as "started"
+update w2l_template_upload_jobs
+set status='dead'
+where status='started'
+  and created_at < ( CURDATE() - INTERVAL 1 DAY );
+
+-- Kill pending jobs with tiktok URLs:
+-- These need to be blocked in the server and job directly.
+update w2l_template_upload_jobs
+set status='dead'
+where status='pending'
+  and download_url LIKE '%tiktok%';

_sql/migrations/2021-05-15-071534_users/up.sql

@@ -0,0 +1,238 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- NB: See the "squashed" copy of this for better documentation
+-- and the current state of the table.
+
+CREATE TABLE users (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Visible "primary key"
+  token VARCHAR(32) NOT NULL,
+
+  -- Username is a lookup key; display_name allows the user to add custom case.
+  username VARCHAR(20) NOT NULL,
+  display_name VARCHAR(20) NOT NULL,
+
+  email_address VARCHAR(255) NOT NULL,
+  email_confirmed BOOLEAN NOT NULL DEFAULT false,
+
+  -- The role assigned to the user confers permissions.
+  user_role_slug VARCHAR(16) NOT NULL,
+
+  -- ========== CREDENTIALS ==========
+
+  -- Bcrypt password hash. Granted, there are newer methods:
+  -- https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html
+  -- Passwords may be a max of 64 characters per BCrypt.
+  password_hash BINARY(60) NOT NULL,
+
+  -- Incremented with every update to the password.
+  password_version INT NOT NULL DEFAULT 0,
+
+  -- ========== ABUSE TRACKING ==========
+
+  -- For abuse tracking.
+  -- Wide enough for IPv4/6
+  ip_address_creation VARCHAR(40) NOT NULL,
+  ip_address_last_login VARCHAR(40) NOT NULL,
+  ip_address_last_update VARCHAR(40) NOT NULL,
+
+  -- ========== DISPLAY / PROFILE ==========
+
+  -- Gravatar image hashes are precomputed.
+  email_gravatar_hash CHAR(32) NOT NULL,
+
+  -- The profile of the user in markdown (user editable).
+  profile_markdown TEXT NOT NULL,
+
+  -- Generated HTML (not user-editable).
+  profile_rendered_html TEXT NOT NULL,
+
+  -- An uploaded avatar. Public hash in our bucket.
+  -- avatar_public_bucket_hash CHAR(32) DEFAULT NULL,
+
+  -- ========== SOCIAL MEDIA ==========
+
+  -- Social media usernames
+  -- These are not confirmed. We'll need to build an OAuth system to handle that.
+  discord_username VARCHAR(36) DEFAULT NULL,
+  twitter_username VARCHAR(36) DEFAULT NULL,
+  twitch_username VARCHAR(36) DEFAULT NULL,
+  patreon_username VARCHAR(36) DEFAULT NULL,
+  github_username VARCHAR(36) DEFAULT NULL,
+  cashapp_username VARCHAR(36) DEFAULT NULL,
+  website_url VARCHAR(255) DEFAULT NULL,
+
+  -- ========== USER PREFERENCES ==========
+
+  -- If the user doesn't want to use gravatar and doesn't have an uploaded avatar.
+  disable_gravatar BOOLEAN NOT NULL DEFAULT false,
+
+  -- (THIS MIGHT NOT BE USED)
+  -- NB: DO NOT SORT!
+  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
+  -- Visibility preference used at TTS inference time (which can be changed on the result itself).
+  -- Moderators will still see them.
+  preferred_tts_result_visibility ENUM(
+    'public',
+    'hidden',
+    'private'
+  ) NOT NULL DEFAULT 'public',
+
+  -- (THIS MIGHT NOT BE USED)
+  -- NB: DO NOT SORT!
+  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
+  -- Visibility preference used at TTS inference time (which can be changed on the result itself).
+  -- Moderators will still see them.
+  preferred_w2l_result_visibility ENUM(
+    'public',
+    'hidden',
+    'private'
+  ) NOT NULL DEFAULT 'public',
+
+  -- Auto play preferences
+  -- Tri-state because we want to potentially change the default behavior.
+  auto_play_audio_preference BOOLEAN DEFAULT NULL,
+  auto_play_video_preference BOOLEAN DEFAULT NULL,
+
+  -- Favorite TTS voice to use by default
+  -- maybe_preferred_tts_model_token VARCHAR(32) DEFAULT NULL,
+
+  -- Favorite W2L model to use by default
+  -- maybe_preferred_w2l_template_token VARCHAR(32) DEFAULT NULL,
+
+  -- Settings
+  -- DO NOT REORDER.
+  -- dark_mode_preference ENUM(
+  --   'light-mode',
+  --   'dark-mode',
+  --   'use-clock'
+  --  ) NOT NULL DEFAULT 'light-mode',
+
+  -- ========== STATS ==========
+
+  -- For tracking stats.
+  -- The "cached" values are updated by a background job.
+  -- cached_tts_rendered_counter INT(10) NOT NULL DEFAULT 0,
+  -- cached_w2l_rendered_counter INT(10) NOT NULL DEFAULT 0,
+
+  -- ========== MODERATION DETAILS ==========
+
+  -- Different than deleted.
+  -- Users still show up, but can't do anything.
+  is_banned BOOLEAN NOT NULL DEFAULT false,
+
+  -- If a moderator has comments.
+  maybe_mod_comments VARCHAR(255) DEFAULT NULL,
+
+  -- The last moderator that made changes.
+  maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
+
+  -- ========== VECTOR CLOCK ==========
+
+  -- Incremented with every update.
+  version INT NOT NULL DEFAULT 0,
+
+  -- ========== TIMESTAMPS ==========
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- If the user is deleted, we set this.
+  -- This is different than banned. These users won't show up at all.
+  user_deleted_at TIMESTAMP NULL,
+  mod_deleted_at TIMESTAMP NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (token),
+  UNIQUE KEY (username),
+  UNIQUE KEY (email_address),
+  KEY fk_user_role_slug (user_role_slug)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
+
+CREATE TABLE user_roles (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Effective "primary key"
+  slug CHAR(16) NOT NULL,
+
+  name VARCHAR(255) NOT NULL,
+
+  -- Usage
+  can_use_tts BOOLEAN NOT NULL DEFAULT FALSE,
+  can_use_w2l BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_own_tts_results BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_own_w2l_results BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_own_account BOOLEAN NOT NULL DEFAULT FALSE,
+
+  -- Contribution
+  can_upload_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
+  can_upload_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_own_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_own_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
+
+  -- Moderation
+  can_approve_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
+  -- Edit includes "lock_from_use" and "lock_from_modification"
+  can_edit_other_users_profiles BOOLEAN NOT NULL DEFAULT FALSE,
+  can_edit_other_users_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
+  can_edit_other_users_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_other_users_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_other_users_tts_results BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_other_users_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_other_users_w2l_results BOOLEAN NOT NULL DEFAULT FALSE,
+  can_ban_users BOOLEAN NOT NULL DEFAULT FALSE,
+  can_delete_users BOOLEAN NOT NULL DEFAULT FALSE,
+
+  -- Incremented with every update.
+  version INT NOT NULL DEFAULT 0,
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (slug)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
+
+CREATE TABLE user_sessions (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Session entropy
+  token VARCHAR(32) NOT NULL,
+
+  -- Foreign key to user
+  user_token VARCHAR(32) NOT NULL,
+
+  -- Track each session's creation IP.
+  -- Wide enough for IPv4/6
+  ip_address_creation VARCHAR(40) NOT NULL,
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- Session termination time.
+  -- This must be set by the server code, or the session is invalid
+  expires_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+  -- deletion = session termination
+  -- Typically these are destroyed by users, but if in the future we allow mods to
+  -- delete them, it doesn't really matter who did the deletion: sessions are not
+  -- designed to be recoverable.
+  deleted_at TIMESTAMP NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (token),
+  KEY fk_user_token (user_token),
+  KEY index_ip_address_creation (ip_address_creation)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;