mirror of
https://github.com/storytold/artcraft.git
synced 2026-10-26 03:39:28 -04:00
Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Paul Sajna
|
81e6c432e4 | ||
|
Paul Sajna
|
73db4a8f40 | ||
|
Paul Sajna
|
925860bbbc | ||
|
Paul Sajna
|
6464172306 | ||
|
Paul Sajna
|
0af881ca5f | ||
|
Paul Sajna
|
7594a90abb | ||
|
Paul Sajna
|
1e07dda1e6 | ||
|
Paul Sajna
|
0804bba121 | ||
|
Paul Sajna
|
4483aa256e | ||
|
Paul Sajna
|
cc0e84d0a5 | ||
|
Paul Sajna
|
8a525f31a4 | ||
|
Paul Sajna
|
b571ccf1c2 | ||
|
Paul Sajna
|
86040a3764 |
@@ -1,11 +1,9 @@
|
||||
*secret*sh
|
||||
*~
|
||||
.dockerignore
|
||||
.env
|
||||
.env-secrets
|
||||
.git
|
||||
.idea/
|
||||
Dockerfile
|
||||
__pycache__/
|
||||
_ignored_schema.rs
|
||||
models/Wav2Lip/python/
|
||||
|
||||
@@ -1,14 +1,7 @@
|
||||
SERVER_ENVIRONMENT=development
|
||||
|
||||
DATABASE_URL=mysql://storyteller:password@localhost/storyteller
|
||||
#DATABASE_URL=mysql://root:root@localhost/storyteller
|
||||
#DATABASE_URL=mysql://root@localhost/storyteller
|
||||
|
||||
# Diesel-cli reads this for the migration directory to use.
|
||||
# We've renamed it so the migrations are sorted near the top of the project directory tree
|
||||
# (since they kind of serve as documentation)
|
||||
MIGRATION_DIRECTORY=_sql/migrations/
|
||||
|
||||
# SQLX normally requires a live database to reflect upon at build time.
|
||||
# This setting turns that on and requires a `sql-data.json` cache file instead.
|
||||
# See https://github.com/launchbadge/sqlx/tree/master/sqlx-cli#readme
|
||||
@@ -71,14 +64,3 @@ TTS_HIFIGAN_SUPERRES_VOCODER_MODEL_FILENAME=hifigan_superres_twilight_33000.pt
|
||||
|
||||
# Development key. Permissible to check in.
|
||||
NEWRELIC_API_KEY=c402be1c4c4216ec83d0cfc0851520cfFFFFNRAL
|
||||
|
||||
# Stripe configs
|
||||
# All secrets are development and okay to check in
|
||||
STRIPE_CHECKOUT_SUCCESS_URL_PATH=/checkout_success
|
||||
STRIPE_CHECKOUT_CANCEL_URL_PATH=/checkout_cancel
|
||||
STRIPE_PORTAL_RETURN_URL_PATH=/portal_success
|
||||
STRIPE_PORTAL_DEFAULT_CONFIG_ID=bpc_1LyPPREU5se17MekYiViZF12
|
||||
# STRIPE_PORTAL_DEFAULT_CONFIG_ID=bpc_1M4jc3EU5se17MekmgdS4dDJ # Production (not a secret)
|
||||
STRIPE_PUBLISHABLE_KEY=pk_test_51KZvWXEU5se17Mekx6e0gw32ovDF78k2eeaGctLIs8oQVxN6DerEp1BDqh6gLMYKli4VsTZoVGHoxq5RmwKMlxtP00r59UEEad
|
||||
STRIPE_SECRET_KEY=sk_test_51KZvWXEU5se17Mek11ZdnWr4frnyw7tWOPuvl3pRzL290zvkT6KqcbGUl3fcvc3hA8oH6lWXz2kGWXXUlNcCuDxB003uVT9JX0
|
||||
STRIPE_SECRET_WEBHOOK_SIGNING_KEY=whsec_40b60bd75768052532b63dbeefa7a1b163e182c3e657fb27fa906ef49a2cd59f
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
# Environmental configurations specific to `download-job`.
|
||||
# These are defaulted for dev and are easy to overwrite in production with K8s environment variables.
|
||||
# !! DOES *NOT* CONTAIN SECRETS !!
|
||||
|
||||
WEB_DOWNLOADER_COMMAND=./download_internet_file.py
|
||||
WEB_DOWNLOADER_MAYBE_VENV_SCRIPT=./python/bin/activate
|
||||
WEB_DOWNLOADER_MAYBE_DOCKER_IMAGE=d73f28ce3ff6
|
||||
|
||||
SOFTVC_MODEL_CHECK_ROOT_DIRECTORY=/
|
||||
SOFTVC_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate"
|
||||
SOFTVC_MODEL_CHECK_MAYBE_DOCKER_IMAGE=a79106991ce3
|
||||
|
||||
TACOTRON_MODEL_CHECK_ROOT_DIRECTORY=/models/tts
|
||||
TACOTRON_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate"
|
||||
TACOTRON_MODEL_CHECK_COMMAND=./vocodes_model_check_tacotron.py
|
||||
TACOTRON_MODEL_CHECK_MAYBE_DOCKER_IMAGE=5642d0fd7fc1
|
||||
|
||||
HIFIGAN_SOFTVC_MODEL_CHECK_ROOT_DIRECTORY=/
|
||||
HIFIGAN_SOFTVC_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate"
|
||||
HIFIGAN_SOFTVC_MODEL_CHECK_MAYBE_DOCKER_IMAGE=a79106991ce3
|
||||
|
||||
HIFIGAN_MODEL_CHECK_ROOT_DIRECTORY=/models/tts
|
||||
HIFIGAN_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate"
|
||||
HIFIGAN_MODEL_CHECK_COMMAND=./vocodes_model_check_hifigan.py
|
||||
HIFIGAN_MODEL_CHECK_MAYBE_DOCKER_IMAGE=5642d0fd7fc1
|
||||
@@ -1,5 +0,0 @@
|
||||
# Ignore certain code for Github language statistics
|
||||
# https://github.com/github/linguist/blob/master/docs/overrides.md
|
||||
old_code/trumpet-trumped.com/dictionary/cmudict-0.7b linguist-vendored
|
||||
old_code/voder-vocodes-1.0/frontends/admin/node_modules linguist-vendored
|
||||
windows/sqlite linguist-vendored
|
||||
@@ -1,52 +0,0 @@
|
||||
name: CI
|
||||
on: [push, pull_request]
|
||||
env:
|
||||
MYSQL_URL: mysql://root:root@localhost/storyteller
|
||||
DATABASE_URL: mysql://root:root@localhost/storyteller
|
||||
REDIS_0_URL: redis://redisd@localhost/0
|
||||
REDIS_1_URL: redis://redisd@localhost/1
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
|
||||
jobs:
|
||||
setup_build_test:
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout sources
|
||||
uses: actions/checkout@v3
|
||||
- name: Cache cargo registry
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
|
||||
- name: Setup mysql
|
||||
run: |
|
||||
sudo systemctl start mysql
|
||||
sudo apt-get update
|
||||
sudo apt-get -y install mysql-client libmysqlclient-dev
|
||||
mysql -e "create user 'storyteller'@'localhost' IDENTIFIED BY 'password';" -uroot -proot
|
||||
mysql -e "create database storyteller; grant all on storyteller.* to 'storyteller'@'localhost';" -uroot -proot
|
||||
- name: Setup Redis
|
||||
run: |
|
||||
sudo apt-get -y install redis-server
|
||||
sudo service redis-server start
|
||||
- name: Setup Rust
|
||||
run: |
|
||||
rustup set profile minimal
|
||||
rustup show
|
||||
rustup install 1.66
|
||||
rustup default 1.66
|
||||
rustup show
|
||||
cargo install sqlx-cli --features openssl-vendored
|
||||
cargo install diesel_cli
|
||||
- name: Run database migrations
|
||||
run: |
|
||||
# nothing to see here ;)
|
||||
set +e +o pipefail
|
||||
diesel migration run
|
||||
set -e -o pipefail
|
||||
- name: Build
|
||||
run: cargo build --release
|
||||
- name: Test
|
||||
run: cargo test --release
|
||||
@@ -15,98 +15,53 @@ env:
|
||||
jobs:
|
||||
# Push image to GitHub Packages.
|
||||
# See also https://docs.docker.com/docker-hub/builds/
|
||||
# See also https://depot.dev/blog/docker-layer-caching-in-github-actions
|
||||
# See also https://evilmartians.com/chronicles/build-images-on-github-actions-with-docker-layer-caching
|
||||
build-docker-image:
|
||||
name: "Build Docker Image"
|
||||
runs-on: ubuntu-22.04 # runs-on: ubuntu-latest
|
||||
push:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'push'
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
# - uses: docker/setup-buildx-action@v1
|
||||
# name: Set up Docker Buildx
|
||||
|
||||
- uses: docker/login-action@v1
|
||||
name: Login to GitHub Container Registry
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }} # username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
#username: ${{ github.repository_owner }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Calculate short SHA
|
||||
id: vars
|
||||
# echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
|
||||
- name: Build image
|
||||
run: |
|
||||
echo "sha_short=$(echo ${{ github.sha }} | cut -c1-12)" >> $GITHUB_OUTPUT
|
||||
SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
|
||||
docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" --build-arg GIT_SHA=${SHORT_SHA}
|
||||
|
||||
- name: Emit short SHA
|
||||
- name: Push image to GitHub Container Registry
|
||||
run: |
|
||||
echo "Full SHA: ${{ github.sha }}"
|
||||
echo "Short SHA: ${{ steps.vars.outputs.sha_short }}"
|
||||
IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
|
||||
|
||||
#- uses: actions/cache@v2
|
||||
# name: Cache Docker layers
|
||||
# with:
|
||||
# path: /tmp/.buildx-cache
|
||||
# key: ${{ runner.os }}-buildx-${{ github.sha }}
|
||||
# restore-keys: |
|
||||
# ${{ runner.os }}-buildx-
|
||||
# Change all uppercase to lowercase
|
||||
IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
|
||||
|
||||
# Docs for this action are at https://github.com/marketplace/actions/build-docker-images-using-cache
|
||||
- uses: whoan/docker-build-with-cache-action@v6
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }} # username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
image_name: storytold/storyteller-rust
|
||||
# NB: The `git_push_tag: true` option does not work as expected, so we put the sha in `image_tag`:
|
||||
# https://github.com/whoan/docker-build-with-cache-action/issues/72
|
||||
image_tag: latest,${{ github.sha }},${{ steps.vars.outputs.sha_short }}
|
||||
# NB: Inject the Git SHA into the container so the servers can emit their own version:
|
||||
build_extra_args: "--build-arg=GIT_SHA=${{ github.sha }}"
|
||||
# Strip git ref prefix from version
|
||||
VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
|
||||
|
||||
# - name: Build image
|
||||
# run: |
|
||||
# SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
|
||||
# docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" --build-arg GIT_SHA=${SHORT_SHA}
|
||||
# Strip "v" prefix from tag name
|
||||
[[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
|
||||
|
||||
# - name: Push image to GitHub Container Registry
|
||||
# run: |
|
||||
# IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
|
||||
# Use Docker `latest` tag convention
|
||||
[ "$VERSION" == "$default-branch" ] && VERSION=latest
|
||||
|
||||
# # Change all uppercase to lowercase
|
||||
# IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
|
||||
echo IMAGE_ID=$IMAGE_ID
|
||||
echo VERSION=$VERSION
|
||||
|
||||
# # Strip git ref prefix from version
|
||||
# VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
|
||||
docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
|
||||
docker push $IMAGE_ID:$VERSION
|
||||
|
||||
# # Strip "v" prefix from tag name
|
||||
# [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
|
||||
SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
|
||||
|
||||
# # Use Docker `latest` tag convention
|
||||
# [ "$VERSION" == "$default-branch" ] && VERSION=latest
|
||||
docker tag $IMAGE_NAME $IMAGE_ID:$SHORT_SHA
|
||||
docker push $IMAGE_ID:$SHORT_SHA
|
||||
|
||||
# echo IMAGE_ID=$IMAGE_ID
|
||||
# echo VERSION=$VERSION
|
||||
|
||||
# docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
|
||||
# docker push $IMAGE_ID:$VERSION
|
||||
|
||||
# SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12)
|
||||
|
||||
# docker tag $IMAGE_NAME $IMAGE_ID:$SHORT_SHA
|
||||
# docker push $IMAGE_ID:$SHORT_SHA
|
||||
|
||||
- name: Cleanup / Debug
|
||||
# This job will run even on failures to help with debugging.
|
||||
if: always()
|
||||
run: |
|
||||
echo "Report at final github action step:"
|
||||
echo "Disk usage:"
|
||||
pwd
|
||||
du -hsc * | sort -hr
|
||||
|
||||
+11
-40
@@ -1,51 +1,22 @@
|
||||
# Secrets are BAD!
|
||||
*.swo
|
||||
*.swp
|
||||
*secret*sh
|
||||
*~
|
||||
.env
|
||||
.env-aichatbot-secrets
|
||||
.env-secrets
|
||||
.idea/
|
||||
__pycache__/
|
||||
_ignored_schema.rs
|
||||
db.secrets
|
||||
db_secrets.toml
|
||||
models/Wav2Lip/python/
|
||||
models/tacotron2/python/
|
||||
production_database_secret_connect.sh
|
||||
production_database_secret_query.sh
|
||||
production_database_voice_clone_report_download.sh
|
||||
secrets.toml
|
||||
secrets.yaml
|
||||
_develop/tools/.secrets
|
||||
_develop/tools/db_secrets.toml
|
||||
twitch_secrets.toml
|
||||
|
||||
# Rust build directory
|
||||
target/
|
||||
crates/lib/database_queries/target/
|
||||
crates/lib/sqlite_queries/target/
|
||||
|
||||
# Vim noise
|
||||
*.swo
|
||||
*.swp
|
||||
*~
|
||||
|
||||
# IntelliJ/CLion noise
|
||||
.idea/
|
||||
|
||||
# Python noise
|
||||
__pycache__/
|
||||
|
||||
# We use diesel for DB migrations only. We don't care about its codgen.
|
||||
_ignored_schema.rs
|
||||
|
||||
# AI Chat Sidecar has its own schema managed by diesel, but we don't care for diesel codegen.
|
||||
crates/programs/aichatbot-sidecar/_ignored_schema.rs
|
||||
|
||||
# AI Chat sidecar's SQlite database shouldn't be checked in
|
||||
crates/programs/aichatbot-sidecar/database.db
|
||||
|
||||
# Runtime data for the AI chatbot
|
||||
runtime_data/
|
||||
|
||||
# Old Python virtual environments
|
||||
models/Wav2Lip/python/
|
||||
models/tacotron2/python/
|
||||
|
||||
# Stuff I want to keep around, but not commit
|
||||
scraps/
|
||||
|
||||
tools/.secrets
|
||||
tools/db_secrets.toml
|
||||
twitch_secrets.toml
|
||||
|
||||
Generated
+4
-21
@@ -1,35 +1,18 @@
|
||||
<component name="ProjectDictionaryState">
|
||||
<!--
|
||||
Despite spellcheck dictionaries being created per-user, IntelliJ loads all of them, per:
|
||||
https://stackoverflow.com/a/28151692
|
||||
-->
|
||||
<dictionary name="bt">
|
||||
<words>
|
||||
<w>arpabet</w>
|
||||
<w>backfill</w>
|
||||
<w>cashapp</w>
|
||||
<w>cheermote</w>
|
||||
<w>crockford</w>
|
||||
<w>culted</w>
|
||||
<w>dalle</w>
|
||||
<w>downvote</w>
|
||||
<w>fakeyou</w>
|
||||
<w>ffprobe</w>
|
||||
<w>greenlight</w>
|
||||
<w>greenlighting</w>
|
||||
<w>greenlit</w>
|
||||
<w>hanashi</w>
|
||||
<w>hifigan</w>
|
||||
<w>replayable</w>
|
||||
<w>retryably</w>
|
||||
<w>scrollback</w>
|
||||
<w>softvc</w>
|
||||
<w>symphonia</w>
|
||||
<w>tacotron</w>
|
||||
<w>techcrunch</w>
|
||||
<w>transcoded</w>
|
||||
<w>transcodings</w>
|
||||
<w>upserting</w>
|
||||
<w>vocoded</w>
|
||||
<w>vocoder</w>
|
||||
<w>vocoders</w>
|
||||
<w>vocodes</w>
|
||||
</words>
|
||||
</dictionary>
|
||||
</component>
|
||||
Generated
+651
-3999
File diff suppressed because it is too large
Load Diff
+18
-56
@@ -1,67 +1,29 @@
|
||||
[workspace]
|
||||
|
||||
members = [
|
||||
# Types: common strongly-typed tokens, ids, payloads, etc.
|
||||
# Also common cloud buckets and other "strongly typed" configs.
|
||||
"crates/_types/buckets",
|
||||
"crates/_types/enums",
|
||||
"crates/_types/reusable_types", # TODO(bt, 2022-12-21): Deprecate
|
||||
"crates/_types/tokens",
|
||||
|
||||
# CLI tools for operations, etc.
|
||||
"crates/cli/database_backup",
|
||||
"crates/cli/database_cleanup",
|
||||
"crates/cli/scrape_test",
|
||||
|
||||
# Shared libraries
|
||||
"crates/lib/actix_helpers",
|
||||
"crates/lib/caching/memory_caching",
|
||||
"crates/lib/caching/redis_caching",
|
||||
"crates/lib/clients/cloud_storage",
|
||||
"crates/lib/clients/fakeyou_client",
|
||||
"crates/lib/clients/google_drive_common",
|
||||
"crates/lib/clients/twitch_common",
|
||||
"crates/lib/concurrency",
|
||||
"crates/lib/config", # TODO(bt, 2023-02-13): Deprecate or repurpose
|
||||
"crates/lib/container_common", # TODO(bt, 2023-02-13): Pull apart and rehome pieces
|
||||
"crates/lib/crockford",
|
||||
"crates/lib/database_queries", # TODO(bt2, 2023-02-13): Rename "mysql_queries", though this will not live beyond a single DB
|
||||
"crates/lib/datetimes",
|
||||
"crates/lib/errors",
|
||||
"crates/lib/files",
|
||||
"crates/lib/hashing",
|
||||
"crates/lib/config",
|
||||
"crates/lib/container_common",
|
||||
"crates/lib/database_queries",
|
||||
"crates/lib/google_drive_common",
|
||||
"crates/lib/http_server_common",
|
||||
"crates/lib/idempotency",
|
||||
"crates/lib/jobs_common",
|
||||
"crates/lib/media",
|
||||
"crates/lib/memory_caching",
|
||||
"crates/lib/redis_common",
|
||||
"crates/lib/sqlite_queries",
|
||||
"crates/lib/sqlx_mysql_helpers",
|
||||
"crates/lib/subprocess_common",
|
||||
"crates/lib/reusable_types",
|
||||
"crates/lib/storage_buckets_common",
|
||||
"crates/lib/tts_common",
|
||||
"crates/lib/url_config",
|
||||
"crates/lib/twitch_common",
|
||||
"crates/lib/user_input_common",
|
||||
"crates/lib/web_scrapers",
|
||||
|
||||
# Programs: Desktop apps
|
||||
"crates/programs/aichatbot-sidecar",
|
||||
|
||||
# Services: API servers, jobs, workers, etc.
|
||||
"crates/service/job/analytics_job",
|
||||
"crates/service/job/download_job",
|
||||
"crates/service/job/inference_job",
|
||||
"crates/service/job/tts_download_job",
|
||||
"crates/service/job/tts_inference_job",
|
||||
"crates/service/job/w2l_download_job",
|
||||
"crates/service/job/w2l_inference_job",
|
||||
"crates/service/other/twitch_pubsub_subscriber",
|
||||
"crates/service/web/dummy_service",
|
||||
"crates/service/web/image_generation_service",
|
||||
"crates/service/web/storyteller_web",
|
||||
"crates/service/web/websocket_gateway",
|
||||
|
||||
# Service Plugins: complete subsystems that can be plugged into services (users, billing, etc.)
|
||||
"crates/service/plugins/billing",
|
||||
"crates/service/plugins/inference_workloads",
|
||||
"crates/service/plugins/users",
|
||||
# Application
|
||||
"crates/service/generic_download_job",
|
||||
"crates/service/storyteller_web",
|
||||
"crates/service/tts_download_job",
|
||||
"crates/service/tts_inference_job",
|
||||
"crates/service/twitch_pubsub_subscriber",
|
||||
"crates/service/w2l_download_job",
|
||||
"crates/service/w2l_inference_job",
|
||||
"crates/service/websocket_gateway",
|
||||
]
|
||||
|
||||
|
||||
+111
-152
@@ -1,104 +1,70 @@
|
||||
# ================================================================
|
||||
# =============== (1) set up core rust build image ===============
|
||||
# ================================================================
|
||||
# ==================== Python Build Base ====================
|
||||
|
||||
FROM ghcr.io/storytold/docker-base-images-nvidia-cuda-experimental:080f96bc7087 as pybuild-base
|
||||
|
||||
# The downloader needs youtube-dl as a python library
|
||||
# NB: No longer maintained: https://news.ycombinator.com/item?id=28319624
|
||||
#RUN pip3 install --upgrade youtube-dl
|
||||
|
||||
# Used by the downloader script.
|
||||
# And it looks like youtube-dl is dead...
|
||||
# Per the README, it looks like pip will always pull the latest version.
|
||||
# https://github.com/yt-dlp/yt-dlp#update
|
||||
RUN python3 -m pip install --upgrade yt-dlp
|
||||
|
||||
# ==================== Python Build Step 2: Wav2Lip Requirements ====================
|
||||
|
||||
FROM pybuild-base as pybuild-requirements
|
||||
|
||||
WORKDIR /
|
||||
|
||||
COPY models/Wav2Lip ./models/Wav2Lip
|
||||
WORKDIR models/Wav2Lip
|
||||
|
||||
# NB: We need Python3.6
|
||||
RUN python3.6 -m venv python
|
||||
#RUN virtualenv -p /usr/bin/python3.6 ... wait
|
||||
RUN . python/bin/activate \
|
||||
&& pip install --upgrade pip \
|
||||
&& pip install -r requirements.txt \
|
||||
&& deactivate
|
||||
|
||||
WORKDIR /
|
||||
|
||||
COPY models/tts ./models/tts
|
||||
WORKDIR models/tts
|
||||
|
||||
# NB: Not sure if we need Python3.6 for Tacotron2, but Python3.8 gave me trouble.
|
||||
# NB: Setuptools fix: https://github.com/tensorflow/tensorflow/issues/34302#issuecomment-554450289
|
||||
RUN python3.6 -m venv python
|
||||
RUN . python/bin/activate \
|
||||
&& pip install --upgrade pip setuptools \
|
||||
&& pip install -r requirements-tacotron-python36.txt \
|
||||
&& deactivate
|
||||
|
||||
# ==================== Rust Build Base ====================
|
||||
|
||||
# Custom base image
|
||||
# Make sure to add this repository so it has read acces to the base image:
|
||||
# https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access
|
||||
# FROM ghcr.io/storytold/docker-base-images-rust-ssl:d94ce4350c3b as rust-build
|
||||
FROM ubuntu:jammy as rust-base
|
||||
|
||||
# NB: This can be "stable" or another version.
|
||||
ARG RUST_TOOLCHAIN="1.66.0"
|
||||
|
||||
FROM ghcr.io/storytold/docker-base-images-rust-ssl:latest as rust-build
|
||||
WORKDIR /tmp
|
||||
|
||||
# NB: cmake is required for freetype-sys-0.13.1, which in turn has only been added for egui.
|
||||
# NB: fontconfig is required by servo-fontconfig-sys, which is in the dependency chain for egui.
|
||||
# NB: libfontconfig-dev is required by servo-fontconfig-sys, which is in the dependency chain for egui.
|
||||
# NB: pkg-config and libssl are for container TLS; we may switch to rustls in the future.
|
||||
RUN apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y \
|
||||
build-essential \
|
||||
cmake \
|
||||
curl \
|
||||
fontconfig \
|
||||
libfontconfig1-dev \
|
||||
libssl-dev \
|
||||
pkg-config
|
||||
|
||||
# NB: Fix for fontconfig (servo-fontconfig-sys): https://github.com/alacritty/alacritty/issues/4423#issuecomment-727277235
|
||||
# TODO(bt, 2023-02-23): This has not been verified to work yet.
|
||||
RUN export PKG_CONFIG_PATH=/usr/lib/x86_64-linux-gnu/pkgconfig
|
||||
|
||||
# Install Rust
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
|
||||
| sh -s -- --default-toolchain $RUST_TOOLCHAIN -y
|
||||
|
||||
# Install correct Rust version
|
||||
#RUN $HOME/.cargo/bin/rustup install $RUST_VERSION
|
||||
#RUN $HOME/.cargo/bin/rustup default $RUST_VERSION
|
||||
|
||||
# Report Rust version for build debugging
|
||||
RUN $HOME/.cargo/bin/rustup show
|
||||
RUN $HOME/.cargo/bin/rustc --version
|
||||
RUN $HOME/.cargo/bin/cargo --version
|
||||
|
||||
# Cargo Chef does Rust build caching: https://github.com/LukeMathWalker/cargo-chef
|
||||
# TODO(bt,2023-03-08): builds are failing with "no space left on device"; disabling build caching
|
||||
#RUN $HOME/.cargo/bin/cargo install cargo-chef --locked
|
||||
|
||||
# ======================================================================
|
||||
# =============== (2) use cargo-chef to "plan" the build ===============
|
||||
# ======================================================================
|
||||
|
||||
FROM rust-base AS planner
|
||||
|
||||
# NB: Copying in everything does not appear to impact cached builds if irrelevant files are changed (at least at this step)
|
||||
# TODO(bt,2023-03-08): builds are failing with "no space left on device"; disabling build caching
|
||||
#COPY . .
|
||||
# TODO(bt,2023-03-08): builds are failing with "no space left on device"; disabling build caching
|
||||
#RUN $HOME/.cargo/bin/cargo chef prepare --recipe-path recipe.json
|
||||
|
||||
# ======================================================================================================
|
||||
# =============== (3) "cook" the libraries (cacheable), then run the app build and tests ===============
|
||||
# ======================================================================================================
|
||||
|
||||
FROM rust-base AS builder
|
||||
|
||||
# TODO(bt,2023-03-08): builds are failing with "no space left on device"; disabling build caching
|
||||
#COPY --from=planner /tmp/recipe.json recipe.json
|
||||
|
||||
# NB: This step builds and caches the dependencies as its own layer.
|
||||
# TODO(bt,2023-03-08): builds are failing with "no space left on device"; disabling build caching
|
||||
#RUN $HOME/.cargo/bin/cargo chef cook --release --recipe-path recipe.json
|
||||
|
||||
# NB: Now we build and test our code.
|
||||
COPY Cargo.lock .
|
||||
COPY Cargo.lock .
|
||||
COPY Cargo.toml .
|
||||
COPY sqlx-data.json .
|
||||
COPY crates/ ./crates
|
||||
COPY db/ ./db
|
||||
COPY test_data/ ./test_data
|
||||
COPY container_includes/ ./container_includes
|
||||
|
||||
# Print a report on disk space
|
||||
#RUN echo "Disk usage at root (before tests):"
|
||||
#RUN du -hsc / | sort -hr
|
||||
RUN echo "Disk usage at current directory (before tests):"
|
||||
RUN pwd
|
||||
RUN du -hsc * | sort -hr
|
||||
RUN $HOME/.cargo/bin/cargo fetch
|
||||
|
||||
ARG GIT_SHA
|
||||
RUN echo -n ${GIT_SHA} > GIT_SHA
|
||||
|
||||
# Run all of the tests
|
||||
#RUN SQLX_OFFLINE=true \
|
||||
# LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
|
||||
# $HOME/.cargo/bin/cargo test
|
||||
|
||||
# Print a report on disk space
|
||||
#RUN echo "Disk usage at root (after tests):"
|
||||
#RUN du -hsc / | sort -hr
|
||||
RUN echo "Disk usage at current directory (after tests):"
|
||||
RUN pwd
|
||||
RUN du -hsc * | sort -hr
|
||||
RUN SQLX_OFFLINE=true \
|
||||
LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
|
||||
$HOME/.cargo/bin/cargo test
|
||||
|
||||
# Build all the binaries.
|
||||
RUN SQLX_OFFLINE=true \
|
||||
@@ -107,30 +73,6 @@ RUN SQLX_OFFLINE=true \
|
||||
--release \
|
||||
--bin storyteller-web
|
||||
|
||||
RUN SQLX_OFFLINE=true \
|
||||
LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
|
||||
$HOME/.cargo/bin/cargo build \
|
||||
--release \
|
||||
--bin dummy-service
|
||||
|
||||
RUN SQLX_OFFLINE=true \
|
||||
LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
|
||||
$HOME/.cargo/bin/cargo build \
|
||||
--release \
|
||||
--bin analytics-job
|
||||
|
||||
RUN SQLX_OFFLINE=true \
|
||||
LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
|
||||
$HOME/.cargo/bin/cargo build \
|
||||
--release \
|
||||
--bin download-job
|
||||
|
||||
RUN SQLX_OFFLINE=true \
|
||||
LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
|
||||
$HOME/.cargo/bin/cargo build \
|
||||
--release \
|
||||
--bin inference-job
|
||||
|
||||
RUN SQLX_OFFLINE=true \
|
||||
LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
|
||||
$HOME/.cargo/bin/cargo build \
|
||||
@@ -143,6 +85,12 @@ RUN SQLX_OFFLINE=true \
|
||||
--release \
|
||||
--bin w2l-download-job
|
||||
|
||||
RUN SQLX_OFFLINE=true \
|
||||
LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
|
||||
$HOME/.cargo/bin/cargo build \
|
||||
--release \
|
||||
--bin generic-download-job
|
||||
|
||||
RUN SQLX_OFFLINE=true \
|
||||
LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \
|
||||
$HOME/.cargo/bin/cargo build \
|
||||
@@ -167,61 +115,72 @@ RUN SQLX_OFFLINE=true \
|
||||
--release \
|
||||
--bin twitch-pubsub-subscriber
|
||||
|
||||
# Print a report on disk space
|
||||
RUN echo "Disk usage at current directory (after all builds):"
|
||||
RUN pwd
|
||||
RUN du -hsc * | sort -hr
|
||||
|
||||
# =============================================================
|
||||
# =============== (4) construct the final image ===============
|
||||
# =============================================================
|
||||
|
||||
# Final image
|
||||
FROM ubuntu:jammy as final
|
||||
|
||||
# See: https://github.com/opencontainers/image-spec/blob/master/annotations.md
|
||||
LABEL org.opencontainers.image.title='Storyteller Rust'
|
||||
LABEL org.opencontainers.image.authors='bt@brand.io, echelon@gmail.com'
|
||||
LABEL org.opencontainers.image.description='All of the binaries from the Rust monorepo'
|
||||
LABEL org.opencontainers.image.documentation='https://github.com/storytold/storyteller-web'
|
||||
LABEL org.opencontainers.image.source='https://github.com/storytold/storyteller-web'
|
||||
LABEL org.opencontainers.image.url='https://github.com/storytold/storyteller-web'
|
||||
# FROM ubuntu:xenial
|
||||
FROM pybuild-requirements as final
|
||||
|
||||
WORKDIR /
|
||||
|
||||
# Give the container its version so it can report over HTTP.
|
||||
ARG GIT_SHA
|
||||
RUN echo -n ${GIT_SHA} > GIT_SHA
|
||||
COPY scripts/ ./scripts
|
||||
|
||||
# See: https://github.com/opencontainers/image-spec/blob/master/annotations.md
|
||||
LABEL org.opencontainers.image.authors='bt@brand.io, echelon@gmail.com'
|
||||
LABEL org.opencontainers.image.url='https://github.com/storytold/storyteller-web'
|
||||
LABEL org.opencontainers.image.documentation='https://github.com/storytold/storyteller-web'
|
||||
LABEL org.opencontainers.image.source='https://github.com/storytold/storyteller-web'
|
||||
|
||||
# NB(echelon,2022-05-31): NVIDIA revoked some of their keys.
|
||||
# A better long term fix is to remove these dependencies outright and not have ML deps in this repo.
|
||||
# https://developer.nvidia.com/blog/updating-the-cuda-linux-gpg-repository-key/
|
||||
# https://forums.developer.nvidia.com/t/invalid-public-key-for-cuda-apt-repository/212901/11
|
||||
RUN apt-key del 7fa2af80
|
||||
|
||||
# NB: Fix for 'E: Conflicting values set for option Signed-By regarding source'
|
||||
# NB: We need to remove all sources
|
||||
RUN rm $(find /etc/apt | grep cuda)
|
||||
|
||||
#RUN wget https://developer.download.nvidia.com/compute/cuda/repos/$distro/$arch/cuda-keyring_1.0-1_all.deb
|
||||
RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb
|
||||
RUN dpkg -i cuda-keyring_1.0-1_all.deb
|
||||
|
||||
# NB: Comment this out for non-debug images
|
||||
# TODO: Figure out how this is done elsewhere with just the static binaries
|
||||
# Others: `mysql-client`
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y \
|
||||
curl \
|
||||
python3-pip \
|
||||
python3 \
|
||||
wget
|
||||
|
||||
RUN python3 --version
|
||||
|
||||
# TODO: Use venv to do this instead.
|
||||
RUN pip3 install gdown
|
||||
|
||||
# Copy all the binaries.
|
||||
COPY --from=builder /tmp/target/release/storyteller-web /
|
||||
COPY --from=builder /tmp/target/release/dummy-service /
|
||||
COPY --from=builder /tmp/target/release/analytics-job /
|
||||
COPY --from=builder /tmp/target/release/download-job /
|
||||
COPY --from=builder /tmp/target/release/inference-job /
|
||||
COPY --from=builder /tmp/target/release/tts-download-job /
|
||||
COPY --from=builder /tmp/target/release/tts-inference-job /
|
||||
COPY --from=builder /tmp/target/release/w2l-download-job /
|
||||
COPY --from=builder /tmp/target/release/w2l-inference-job /
|
||||
COPY --from=builder /tmp/target/release/websocket-gateway /
|
||||
COPY --from=builder /tmp/target/release/twitch-pubsub-subscriber /
|
||||
COPY --from=rust-build /tmp/target/release/storyteller-web /
|
||||
COPY --from=rust-build /tmp/target/release/tts-download-job /
|
||||
COPY --from=rust-build /tmp/target/release/tts-inference-job /
|
||||
COPY --from=rust-build /tmp/target/release/w2l-download-job /
|
||||
COPY --from=rust-build /tmp/target/release/w2l-inference-job /
|
||||
COPY --from=rust-build /tmp/target/release/websocket-gateway /
|
||||
COPY --from=rust-build /tmp/target/release/twitch-pubsub-subscriber /
|
||||
COPY --from=rust-build /tmp/target/release/generic-download-job /
|
||||
COPY --from=rust-build /tmp/GIT_SHA /
|
||||
|
||||
# SSL certs are required for crypto
|
||||
COPY --from=builder /etc/ssl /etc/ssl
|
||||
COPY --from=rust-build /etc/ssl /etc/ssl
|
||||
|
||||
# Required dynamically linked libraries
|
||||
COPY --from=builder /usr/lib/x86_64-linux-gnu/libssl.* /lib/x86_64-linux-gnu/
|
||||
COPY --from=builder /usr/lib/x86_64-linux-gnu/libcrypto.* /lib/x86_64-linux-gnu/
|
||||
|
||||
# Container includes
|
||||
COPY container_includes/ /container_includes
|
||||
COPY --from=rust-build /lib/x86_64-linux-gnu/libssl.* /lib/x86_64-linux-gnu/
|
||||
COPY --from=rust-build /lib/x86_64-linux-gnu/libcrypto.* /lib/x86_64-linux-gnu/
|
||||
|
||||
# Make sure all the links resolve
|
||||
RUN ldd storyteller-web
|
||||
|
||||
# Without a .env file, Rust crashes "mysteriously" (ugh)
|
||||
RUN touch .env
|
||||
RUN touch .env-download-job
|
||||
RUN touch .env-secrets
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
@@ -1,46 +1,77 @@
|
||||
storyteller-rust
|
||||
================
|
||||
|
||||
Our Rust monorepo, which serves both our API surface area and our downloadable client applications.
|
||||
Our Rust monorepo, which services our
|
||||
API surface area.
|
||||
|
||||
Application overview
|
||||
--------------------
|
||||
|
||||
* [FakeYou.com](https://fakeyou.com) is powered by:
|
||||
* HTTP APIs:
|
||||
* `storyteller-web` - HTTP + database monolith for FakeYou
|
||||
* Asynchronous Jobs:
|
||||
* `tts-download-job` - Async download of models
|
||||
* `tts-inference-job` - Async TTS inference
|
||||
* `w2l-download-job` - Async download of videos and images
|
||||
* `w2l-inference-job` - Async W2L inference
|
||||
|
||||
* [StorytellerStream](https://storyteller.stream) is powered by:
|
||||
* HTTP APIs:
|
||||
* `storyteller-web` - HTTP + database monolith for FakeYou
|
||||
* Websocket events:
|
||||
* `websocket-gateway` - Websocket to pull down Twitch events.
|
||||
* Twitch API integration:
|
||||
* `twitch-pubsub-subscriber` - Dynamically subscribes to Twitch PubSub for "active" users
|
||||
|
||||
Schema and API notes
|
||||
--------------------
|
||||
|
||||
* [Redis Schema Notes README](_docs/redis_schema.md)
|
||||
|
||||
Local development
|
||||
-----------------
|
||||
|
||||
[Local development setup README](_docs/local_development_setup.md)
|
||||
|
||||
Production and Deployment
|
||||
-------------------------
|
||||
|
||||
[Production builds and deployment README](_docs/production_builds_and_deployment.md)
|
||||
|
||||
Development notes
|
||||
-----------------
|
||||
|
||||
[Development notes README](_docs/development_notes.md)
|
||||
|
||||
Dump a MySQL report
|
||||
-------------------
|
||||
|
||||
```
|
||||
."-,.__
|
||||
`. `. ,
|
||||
.--' .._,'"-' `.
|
||||
. .' `'
|
||||
`. / ,'
|
||||
` '--. ,-"'
|
||||
`"` | \
|
||||
-. \, |
|
||||
`--Y.' ___.
|
||||
\ L._, \
|
||||
_., `. < <\ _
|
||||
,' ' `, `. | \ ( `
|
||||
../, `. ` | .\`. \ \_
|
||||
,' ,.. . _.,' ||\l ) '".
|
||||
, ,' \ ,'.-.`-._,' | . _._`.
|
||||
,' / \ \ `' ' `--/ | \ / / ..\
|
||||
.' / \ . |\__ - _ ,'` ` / / `.`.
|
||||
| ' .. `-...-" | `-' / / . `.
|
||||
| / |L__ | | / / `. `.
|
||||
, / . . | | / / ` `
|
||||
/ / ,. ,`._ `-_ | | _ ,-' / ` \
|
||||
/ . \"`_/. `-_ \_,. ,' +-' `-' _, ..,-. \`.
|
||||
. ' .-f ,' ` '. \__.---' _ .' ' \ \
|
||||
' / `.' l .' / \.. ,_|/ `. ,'` L`
|
||||
|' _.-""` `. \ _,' ` \ `.___`.'"`-. , | | | \
|
||||
|| ,' `. `. ' _,...._ ` | `/ ' | ' .|
|
||||
|| ,' `. ;.,.---' ,' `. `.. `-' .-' /_ .' ;_ ||
|
||||
|| ' V / / ` | ` ,' ,' '. ! `. ||
|
||||
||/ _,-------7 ' . | `-' l / `||
|
||||
. | ,' .- ,' || | .-. `. .' ||
|
||||
`' ,' `".' | | `. '. -.' `'
|
||||
/ ,' | |,' \-.._,.'/'
|
||||
. / . . \ .''
|
||||
.`. | `. / :_,'.'
|
||||
\ `...\ _ ,'-. .' /_.-' S T O R Y T E L L E R
|
||||
`-.__ `, `' . _.>----''. _ __ / Turning memes into dreams
|
||||
.' /"' | "' '_ One Rust Monorepo at a time
|
||||
/_|.-'\ ,". '.'`__'-( \
|
||||
/ ,"'"\,' `/ `-.|"
|
||||
```
|
||||
mysql -u storyteller \
|
||||
-pPasswordHere \
|
||||
-h IP_HERE \
|
||||
-P 3306 \
|
||||
-D storyteller \
|
||||
-B \
|
||||
-e "select * from voice_clone_requests;" | sed "s/'/\'/;s/\t/\",\"/g;s/^/\"/;s/$/\"/;s/\n//g" > voice_clone_report.csv
|
||||
```
|
||||
|
||||
TODO
|
||||
----
|
||||
|
||||
* Revise this README to be more useful.
|
||||
|
||||
Notes / TODOs:
|
||||
|
||||
* Examples for good Actix+Sqlx Tests:
|
||||
https://stackoverflow.com/questions/65370752/how-do-i-create-an-actix-web-server-that-accepts-both-sqlx-database-pools-and-tr
|
||||
|
||||
* Actix/sqlx runtime compat:
|
||||
https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734
|
||||
|
||||
* Redis caching of more endpoints
|
||||
|
||||
* Jobs for analytics queries (#uses per model, etc)
|
||||
* Jobs for ingesting Discord, Twitter, Reddit RPAN chat, Twitch chat (not PubSub), etc.
|
||||
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
AI Chatbot Sidecar
|
||||
==================
|
||||
|
||||
This is the first app that doesn't run as a part of our SoA API architecture. (Though it is written such that we
|
||||
could port it.) This app is responsible for generating all the content that goes into our streaming AI-powered TV
|
||||
programs.
|
||||
|
||||
### Windows Installation
|
||||
|
||||
The application uses sqlite, which is tricky to install on Windows.
|
||||
|
||||
1. Put together an sqlite build directory containing the sources and dll file. One is checked in under this monorepo
|
||||
in `./windows/sqlite` at the time of this writing, though that may change in the future.
|
||||
|
||||
|
||||
2. Add environment variables for the sqlite build directory (see https://github.com/diesel-rs/diesel/issues/487)
|
||||
|
||||
* PATH (append the sqlite build directory to the end)
|
||||
* SQLITE3_LIB_DIR (a new environment variable; set it to the sqllite build path)
|
||||
|
||||
|
||||
3. Build the `sqlite3.lib` file with the following:
|
||||
|
||||
```
|
||||
lib /def:sqlite3.def /out:sqlite3.lib /MACHINE:amd64
|
||||
```
|
||||
|
||||
This must be done in "Developer Command Prompt for VS 2022" (or similar), not `cmd` or `PowerShell`.
|
||||
|
||||
|
||||
4. Run the diesel_cli tool build
|
||||
```
|
||||
cargo install diesel_cli --no-default-features --features sqlite
|
||||
```
|
||||
|
||||
If it still complains about the `sqlite3.lib` file not being found, be sure to double check that `echo %PATH%` shows
|
||||
the updated PATH from the steps above.
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
# Local Kubernetes environment for standing up the APIs and their dependencies
|
||||
|
||||
- Install minikube, kubectl, docker, and musl from your OS package manager
|
||||
- `rustup target add x86_64-unknown-linux-musl`
|
||||
- `cargo build --release --bin storyteller-web --target=x86_64-unknown-linux-musl`
|
||||
- `minikube start --extra-config=apiserver.service-node-port-range=1-65535`
|
||||
- `eval $(minikube docker-env)`
|
||||
- `minikube mount /path/to/storyteller-rust/:/storyteller-rust &> /dev/null &`
|
||||
- `docker build -t storyteller-web -f localdev/Dockerfile .`
|
||||
- `kubectl apply -f localdev/kubernetes.yml`
|
||||
- Get the ip address of minikube with `minikube ip`, this is where all the services will be running on their
|
||||
regular ports (12345 for the api, 6379 for redis, 3306 for mysql)
|
||||
- That's it, we just spun up mysql, applied database migrations, spun up redis, and spun up our api
|
||||
|
||||
# Development tips
|
||||
- Rebuild and "deploy" (locally) with `cargo build --release --bin storyteller-web --target=x86_64-unknown-linux-musl && kubectl rollout restart storyteller-web`
|
||||
- View logs from the api with `kubectl logs storyteller-web<TAB>`
|
||||
- Delete everything minikube (if you start running out of disk space or what-have-you) with `minikube delete`
|
||||
- See pod status with `kubectl get pods`
|
||||
- If there are errors you can get more info with `kubectl describe pods`
|
||||
@@ -1,77 +0,0 @@
|
||||
old readme
|
||||
==========
|
||||
|
||||
Contents of the main README.md, which will always be woefully out of date. Docs will migrate into this folder on a
|
||||
per-subject basis. App-specific docs will live under each app.
|
||||
|
||||
Application overview
|
||||
--------------------
|
||||
|
||||
* [FakeYou.com](https://fakeyou.com) is powered by:
|
||||
* HTTP APIs:
|
||||
* `storyteller-web` - HTTP + database monolith for FakeYou
|
||||
* Asynchronous Jobs:
|
||||
* `tts-download-job` - Async download of models
|
||||
* `tts-inference-job` - Async TTS inference
|
||||
* `w2l-download-job` - Async download of videos and images
|
||||
* `w2l-inference-job` - Async W2L inference
|
||||
|
||||
* [StorytellerStream](https://storyteller.stream) is powered by:
|
||||
* HTTP APIs:
|
||||
* `storyteller-web` - HTTP + database monolith for FakeYou
|
||||
* Websocket events:
|
||||
* `websocket-gateway` - Websocket to pull down Twitch events.
|
||||
* Twitch API integration:
|
||||
* `twitch-pubsub-subscriber` - Dynamically subscribes to Twitch PubSub for "active" users
|
||||
|
||||
Schema and API notes
|
||||
--------------------
|
||||
|
||||
* [Redis Schema Notes README](_docs/redis_schema.md)
|
||||
|
||||
Local development
|
||||
-----------------
|
||||
|
||||
[Local development setup README](_docs/local_development_setup.md)
|
||||
|
||||
Production and Deployment
|
||||
-------------------------
|
||||
|
||||
[Production builds and deployment README](_docs/production_builds_and_deployment.md)
|
||||
|
||||
Development notes
|
||||
-----------------
|
||||
|
||||
[Development notes README](_docs/development_notes.md)
|
||||
|
||||
Dump a MySQL report
|
||||
-------------------
|
||||
|
||||
```
|
||||
mysql -u storyteller \
|
||||
-pPasswordHere \
|
||||
-h IP_HERE \
|
||||
-P 3306 \
|
||||
-D storyteller \
|
||||
-B \
|
||||
-e "select * from voice_clone_requests;" | sed "s/'/\'/;s/\t/\",\"/g;s/^/\"/;s/$/\"/;s/\n//g" > voice_clone_report.csv
|
||||
```
|
||||
|
||||
TODO
|
||||
----
|
||||
|
||||
* Revise this README to be more useful.
|
||||
|
||||
Notes / TODOs:
|
||||
|
||||
* Examples for good Actix+Sqlx Tests:
|
||||
https://stackoverflow.com/questions/65370752/how-do-i-create-an-actix-web-server-that-accepts-both-sqlx-database-pools-and-tr
|
||||
|
||||
* Actix/sqlx runtime compat:
|
||||
https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734
|
||||
|
||||
* Redis caching of more endpoints
|
||||
|
||||
* Jobs for analytics queries (#uses per model, etc)
|
||||
* Jobs for ingesting Discord, Twitter, Reddit RPAN chat, Twitch chat (not PubSub), etc.
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
FROM rust:latest
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y \
|
||||
mariadb-client \
|
||||
imagemagick \
|
||||
libmariadb-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
RUN cargo install diesel_cli --features mysql && \
|
||||
cargo install sqlx-cli --features mysql
|
||||
|
||||
WORKDIR /storyteller-rust
|
||||
|
||||
ARG GIT_SHA
|
||||
|
||||
RUN echo -n ${GIT_SHA} > /GIT_SHA
|
||||
|
||||
CMD ["/storyteller-rust/localdev/startup.sh"]
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: mysql
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: mysql
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: mysql
|
||||
spec:
|
||||
containers:
|
||||
- name: mysql
|
||||
image: mysql:8.0.30
|
||||
env:
|
||||
- name: MYSQL_USER
|
||||
value: storyteller
|
||||
- name: MYSQL_PASSWORD
|
||||
value: password
|
||||
- name: MYSQL_DATABASE
|
||||
value: storyteller
|
||||
- name: MYSQL_ROOT_PASSWORD
|
||||
value: password
|
||||
ports:
|
||||
- containerPort: 3306
|
||||
|
||||
---
|
||||
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: redis
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: redis
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: redis
|
||||
spec:
|
||||
containers:
|
||||
- name: redis
|
||||
image: redis:7.0.4
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
|
||||
---
|
||||
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: storyteller-web
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: storyteller-web
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: storyteller-web
|
||||
spec:
|
||||
containers:
|
||||
- name: storyteller-web
|
||||
image: storyteller-web:latest
|
||||
imagePullPolicy: Never
|
||||
env:
|
||||
- name: MYSQL_URL
|
||||
value: mysql://storyteller:password@mysql.default/storyteller
|
||||
- name: REDIS_0_URL
|
||||
value: redis://redisd@redis.default/0
|
||||
- name: REDIS_1_URL
|
||||
value: redis://redisd@redis.default/1
|
||||
- name: ACCESS_KEY
|
||||
value: garbage
|
||||
- name: SECRET_KEY
|
||||
value: garbage
|
||||
- name: REGION_NAME
|
||||
value: garbage
|
||||
- name: W2L_PRIVATE_DOWNLOAD_BUCKET_NAME
|
||||
value: garbage
|
||||
- name: W2L_PUBLIC_DOWNLOAD_BUCKET_NAME
|
||||
value: garbage
|
||||
- name: AUDIO_UPLOADS_BUCKET_ROOT
|
||||
value: garbage
|
||||
- name: TWITCH_USERNAME
|
||||
value: garbage
|
||||
- name: TWITCH_STREAM_KEY
|
||||
value: garbage
|
||||
- name: TWITCH_APP_CLIENT_ID
|
||||
value: garbage
|
||||
- name: TWITCH_APP_CLIENT_SECRET
|
||||
value: garbage
|
||||
ports:
|
||||
- containerPort: 12345
|
||||
volumeMounts:
|
||||
- mountPath: /storyteller-rust
|
||||
name: storyteller-rust-folder
|
||||
volumes:
|
||||
- name: storyteller-rust-folder
|
||||
hostPath:
|
||||
path: /storyteller-rust
|
||||
---
|
||||
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: mysql
|
||||
spec:
|
||||
type: NodePort
|
||||
ports:
|
||||
- name: mysql
|
||||
port: 3306
|
||||
nodePort: 3306
|
||||
selector:
|
||||
app: mysql
|
||||
|
||||
---
|
||||
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: redis
|
||||
spec:
|
||||
type: NodePort
|
||||
ports:
|
||||
- name: redis
|
||||
port: 6379
|
||||
nodePort: 6379
|
||||
selector:
|
||||
app: redis
|
||||
|
||||
---
|
||||
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: storyteller-web
|
||||
spec:
|
||||
type: NodePort
|
||||
ports:
|
||||
- name: storyteller-web
|
||||
port: 12345
|
||||
nodePort: 12345
|
||||
selector:
|
||||
app: storyteller-web
|
||||
|
||||
|
||||
@@ -1,258 +0,0 @@
|
||||
# Nginx configs for "jungle.horse" and "api.jungle.horse" (http)
|
||||
#
|
||||
# On Ubuntu Linux, this config is located at the following location:
|
||||
#
|
||||
# - /etc/nginx/sites-enabled/{name-of-site}
|
||||
#
|
||||
# Move this file there, and then restart with,
|
||||
#
|
||||
# sudo service nginx restart
|
||||
#
|
||||
# On Mac, this is located at the following locations:
|
||||
#
|
||||
# - /opt/homebrew/etc/nginx/servers/{name-of-site} (for nginx installed with homebrew)
|
||||
# - /usr/local/etc/nginx/sites-enabled/{name-of-site} (for default nginx install)
|
||||
#
|
||||
#
|
||||
# On Mac, verify configs and restart with,
|
||||
#
|
||||
# sudo nginx -t
|
||||
# sudo nginx -s reload
|
||||
#
|
||||
# ================
|
||||
# SSL Instructions
|
||||
# ================
|
||||
#
|
||||
# You can launch Chrome in a way it disregards self-signed certs and CORS:
|
||||
#
|
||||
# chromium --disable-web-security --ignore-certificate-errors --user-data-dir=~/chromeTemp
|
||||
#
|
||||
# To create self-signed certs on Ubuntu Linux,
|
||||
#
|
||||
# sudo apt-get install openssl
|
||||
#
|
||||
# sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/localhost.key -out /etc/ssl/certs/localhost.crt
|
||||
#
|
||||
# ^ most fields can be blank, but the "Common Name" should be `*.jungle.horse`
|
||||
#
|
||||
# sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/devfakeyou.key -out /etc/ssl/certs/devfakeyou.crt
|
||||
# sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/devstoryteller.key -out /etc/ssl/certs/devstoryteller.crt
|
||||
#
|
||||
# ^ most fields can be blank, but the "Common Name" should be `*.dev.fakeyou.com` and `*.dev.storyteller.io` respectively.
|
||||
#
|
||||
# For more: https://www.cloudsavvyit.com/1306/how-to-create-and-use-self-signed-ssl-on-nginx/
|
||||
#
|
||||
# To create self-signed certs on Mac,
|
||||
#
|
||||
# openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt
|
||||
#
|
||||
# ^ most fields can be blank, but the "Common Name" should be `*.jungle.horse`
|
||||
#
|
||||
# On Mac, load the self signed cert into the keychain with the following command:
|
||||
#
|
||||
# sudo security add-trusted-cert \
|
||||
# -d -r trustRoot \
|
||||
# -k /Library/Keychains/System.keychain /etc/ssl/certs/localhost.crt
|
||||
#
|
||||
# Self signed certs might be considered insecure by the browser. In Firefox,
|
||||
# set about:config `security.tls.insecure_fallback_hosts` to include a list of
|
||||
# permitted domains. You may also need to configure other settings.
|
||||
#
|
||||
#
|
||||
|
||||
# ========================================================
|
||||
# FAKEYOU
|
||||
# ========================================================
|
||||
|
||||
# FakeYou javascript frontend over plain HTTP (launched by Yarn)
|
||||
server {
|
||||
listen 80;
|
||||
server_name dev.fakeyou.com;
|
||||
location / {
|
||||
proxy_set_header Host $host;
|
||||
proxy_pass http://127.0.0.1:7000;
|
||||
proxy_redirect off;
|
||||
}
|
||||
}
|
||||
|
||||
# FakeYou API backend over plain HTTP (Rust app)
|
||||
server {
|
||||
listen 80;
|
||||
server_name api.dev.fakeyou.com;
|
||||
location / {
|
||||
proxy_set_header Host $host;
|
||||
proxy_pass http://127.0.0.1:12345;
|
||||
proxy_redirect off;
|
||||
}
|
||||
}
|
||||
|
||||
# FakeYou javascript frontend over SSL (launched by Yarn)
|
||||
server {
|
||||
listen 443 ssl;
|
||||
listen [::]:443 ssl;
|
||||
ssl_certificate /etc/ssl/certs/devfakeyou.crt;
|
||||
ssl_certificate_key /etc/ssl/private/devfakeyou.key;
|
||||
ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
|
||||
server_name dev.fakeyou.com;
|
||||
location / {
|
||||
proxy_set_header Host $host;
|
||||
proxy_pass http://127.0.0.1:7000;
|
||||
proxy_redirect off;
|
||||
}
|
||||
}
|
||||
|
||||
# FakeYou API backend over SSL (Rust app)
|
||||
server {
|
||||
listen 443 ssl;
|
||||
listen [::]:443 ssl;
|
||||
ssl_certificate /etc/ssl/certs/devfakeyou.crt;
|
||||
ssl_certificate_key /etc/ssl/private/devfakeyou.key;
|
||||
ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
|
||||
server_name api.dev.fakeyou.com;
|
||||
location / {
|
||||
proxy_set_header Host $host;
|
||||
proxy_pass http://127.0.0.1:12345;
|
||||
proxy_redirect off;
|
||||
}
|
||||
}
|
||||
|
||||
# ========================================================
|
||||
# STORYTELLER
|
||||
# ========================================================
|
||||
|
||||
# Storyteller javascript frontend over plain HTTP (launched by Yarn)
|
||||
server {
|
||||
listen 80;
|
||||
server_name dev.storyteller.io;
|
||||
location / {
|
||||
proxy_set_header Host $host;
|
||||
proxy_pass http://127.0.0.1:7000;
|
||||
proxy_redirect off;
|
||||
}
|
||||
}
|
||||
|
||||
# Storyteller API backend over plain HTTP
|
||||
server {
|
||||
listen 80;
|
||||
server_name api.dev.storyteller.io;
|
||||
location / {
|
||||
proxy_set_header Host $host;
|
||||
proxy_pass http://127.0.0.1:12345;
|
||||
proxy_redirect off;
|
||||
}
|
||||
}
|
||||
|
||||
# Storyteller javascript frontend over SSL (launched by Yarn)
|
||||
server {
|
||||
listen 443 ssl;
|
||||
listen [::]:443 ssl;
|
||||
ssl_certificate /etc/ssl/certs/devstoryteller.crt;
|
||||
ssl_certificate_key /etc/ssl/private/devstoryteller.key;
|
||||
ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
|
||||
server_name dev.storyteller.io;
|
||||
location / {
|
||||
proxy_set_header Host $host;
|
||||
proxy_pass http://127.0.0.1:7000;
|
||||
proxy_redirect off;
|
||||
}
|
||||
}
|
||||
|
||||
# Storyteller API backend over SSL
|
||||
server {
|
||||
listen 443 ssl;
|
||||
listen [::]:443 ssl;
|
||||
ssl_certificate /etc/ssl/certs/devstoryteller.crt;
|
||||
ssl_certificate_key /etc/ssl/private/devstoryteller.key;
|
||||
ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
|
||||
server_name api.dev.storyteller.io;
|
||||
location / {
|
||||
proxy_set_header Host $host;
|
||||
proxy_pass http://127.0.0.1:12345;
|
||||
proxy_redirect off;
|
||||
}
|
||||
}
|
||||
|
||||
# ========================================================
|
||||
# OLD PRODUCTS THAT NEED UPDATING
|
||||
# ========================================================
|
||||
|
||||
# ## TODO(2022-02-13): This is new and untested
|
||||
# # WebSocket API backend over plain HTTP
|
||||
# server {
|
||||
# listen 80;
|
||||
# server_name ws.jungle.horse;
|
||||
# location / {
|
||||
# proxy_set_header Host $host;
|
||||
# proxy_pass http://127.0.0.1:54321;
|
||||
# proxy_redirect off;
|
||||
# }
|
||||
# }
|
||||
#
|
||||
# # Websocket API backend over SSL
|
||||
# server {
|
||||
# listen 443 ssl;
|
||||
# listen [::]:443 ssl;
|
||||
# ssl_certificate /etc/ssl/certs/localhost.crt;
|
||||
# ssl_certificate_key /etc/ssl/private/localhost.key;
|
||||
# ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
|
||||
# server_name ws.jungle.horse;
|
||||
# location / {
|
||||
# proxy_set_header Host $host;
|
||||
# proxy_pass http://127.0.0.1:54321;
|
||||
# proxy_redirect off;
|
||||
# }
|
||||
# }
|
||||
#
|
||||
# # Javascript frontend over plain HTTP
|
||||
# server {
|
||||
# listen 80;
|
||||
# server_name jungle.horse;
|
||||
# location / {
|
||||
# proxy_set_header Host $host;
|
||||
# proxy_pass http://127.0.0.1:7000;
|
||||
# proxy_redirect off;
|
||||
# }
|
||||
# }
|
||||
#
|
||||
# # TODO(2022-02-13): This is new and untested
|
||||
# # Javascript frontend (OBS Layer) over plain HTTP
|
||||
# server {
|
||||
# listen 80;
|
||||
# server_name obs.jungle.horse;
|
||||
# location / {
|
||||
# proxy_set_header Host $host;
|
||||
# proxy_pass http://127.0.0.1:7007;
|
||||
# proxy_redirect off;
|
||||
# }
|
||||
# }
|
||||
#
|
||||
# # Javascript frontend over SSL
|
||||
# server {
|
||||
# listen 443 ssl;
|
||||
# listen [::]:443 ssl;
|
||||
# ssl_certificate /etc/ssl/certs/localhost.crt;
|
||||
# ssl_certificate_key /etc/ssl/private/localhost.key;
|
||||
# ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
|
||||
# server_name jungle.horse;
|
||||
# location / {
|
||||
# proxy_set_header Host $host;
|
||||
# proxy_pass http://127.0.0.1:7000;
|
||||
# proxy_redirect off;
|
||||
# }
|
||||
# }
|
||||
#
|
||||
# # Javascript frontend over SSL (OBS Layer)
|
||||
# server {
|
||||
# listen 443 ssl;
|
||||
# listen [::]:443 ssl;
|
||||
# ssl_certificate /etc/ssl/certs/localhost.crt;
|
||||
# ssl_certificate_key /etc/ssl/private/localhost.key;
|
||||
# ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
|
||||
# server_name obs.jungle.horse;
|
||||
# location / {
|
||||
# proxy_set_header Host $host;
|
||||
# proxy_pass http://127.0.0.1:7007;
|
||||
# proxy_redirect off;
|
||||
# }
|
||||
# }
|
||||
#
|
||||
@@ -1,6 +0,0 @@
|
||||
#!/bin/bash
|
||||
set +e +o pipefail
|
||||
diesel migration run --database-url="${MYSQL_URL}"
|
||||
set -e -o pipefail
|
||||
./target/x86_64-unknown-linux-musl/release/storyteller-web
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
echo 'Installing APT packages...'
|
||||
|
||||
# OpenSSL requirements: libssl-dev, pkg-config
|
||||
# Database requirements: mysql-server, libmysqlclient-dev
|
||||
# Other tools: jq (db codegen)
|
||||
sudo apt-get install -y \
|
||||
jq \
|
||||
libmysqlclient-dev \
|
||||
libssl-dev \
|
||||
mysql-server \
|
||||
pkg-config \
|
||||
--no-install-recommends
|
||||
|
||||
echo 'Installing Cargo packages...'
|
||||
|
||||
# Install Diesel CLI for migrations
|
||||
cargo install diesel_cli \
|
||||
--no-default-features \
|
||||
--features mysql
|
||||
|
||||
# Install SQLx for codegen
|
||||
cargo install sqlx-cli \
|
||||
--no-default-features \
|
||||
--features rustls,mysql
|
||||
|
||||
echo 'Installing Database user...'
|
||||
|
||||
# Create DB account (using default system mysql password)
|
||||
sudo mysql \
|
||||
-u "root" \
|
||||
-p"password" \
|
||||
-e "use mysql; CREATE DATABASE IF NOT EXISTS storyteller; CREATE USER IF NOT EXISTS 'storyteller'@'localhost' IDENTIFIED BY 'password'; GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';"
|
||||
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
dev setup
|
||||
=========
|
||||
|
||||
[Back to main README](../../README.md)
|
||||
[Back to main README](../README.md)
|
||||
|
||||
## Debugging and Fixing CUDA/PyTorch
|
||||
### Debugging and Fixing CUDA/PyTorch
|
||||
|
||||
See this spreadsheet for previous battles with version incompatibility:
|
||||
https://docs.google.com/spreadsheets/d/1BEdLmwOzo3r83-iJn9sj6co1VT92t3lIeJ-lNH25bdQ/edit#gid=0
|
||||
@@ -11,53 +11,27 @@ https://docs.google.com/spreadsheets/d/1BEdLmwOzo3r83-iJn9sj6co1VT92t3lIeJ-lNH25
|
||||
Ubuntu might upgrade the driver by accident, and it might need reinstallation. Who knows.
|
||||
This stuff is a nightmare.
|
||||
|
||||
## Database Setup
|
||||
|
||||
### Mac Install
|
||||
|
||||
```
|
||||
brew install mysql
|
||||
# If mysql -uroot fails, reboot the machine:
|
||||
sudo reboot now
|
||||
```
|
||||
|
||||
### Linux Install
|
||||
### Database Setup
|
||||
|
||||
Install the following libraries, and see the notes further below about MySQL on Ubuntu 20.04.
|
||||
|
||||
```
|
||||
jq # for combining mysql codegen outputs
|
||||
libmysqlclient-dev
|
||||
mysql-server
|
||||
libsqlite3-dev
|
||||
pkgconf # if using openssl instead of rustls
|
||||
imagemagick
|
||||
```
|
||||
|
||||
### Migrations
|
||||
|
||||
To manage the database and perform migrations, install the Rust tools diesel and sqlx.
|
||||
|
||||
We'll be using diesel to manage the migrations, but sqlx within the app to actually perform queries.
|
||||
Diesel is an ORM, which is dumb, so we use sqlx as at-compile-time typesafe SQL.
|
||||
|
||||
The CLI is necessary to perform codegen:
|
||||
|
||||
```
|
||||
sudo apt-get install libmysqlclient-dev
|
||||
cargo install sqlx-cli --no-default-features --features rustls,mysql,sqlite
|
||||
cargo install sqlx-cli --no-default-features --features mysql [2022-01-16: is this needed?]
|
||||
```
|
||||
|
||||
Diesel now supports configuring the migration directory via an environment variable,
|
||||
so we can install the currently published version:
|
||||
|
||||
```
|
||||
cargo install diesel_cli \
|
||||
--no-default-features \
|
||||
--features mysql,sqlite
|
||||
```
|
||||
|
||||
If the modern version fails, try installing a cherry-picked version with a
|
||||
TOML-based migration directory configuration:
|
||||
To install a more modern version of Diesel (unreleased) that
|
||||
supports the "migrations_dir" toml config option:
|
||||
|
||||
```
|
||||
cargo install diesel_cli \
|
||||
@@ -86,21 +60,6 @@ If MySql in local dev can't be connected to, reset the accounts:
|
||||
|
||||
https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux
|
||||
|
||||
#### MySql on Ubuntu 22.04
|
||||
|
||||
Should work largely out of the box. We'll need a dev account:
|
||||
|
||||
```
|
||||
# Connect to mysql:
|
||||
# sudo mysql -u root -p (password is "root")
|
||||
|
||||
use mysql;
|
||||
CREATE USER 'storyteller'@'localhost' IDENTIFIED BY 'password';
|
||||
GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';
|
||||
```
|
||||
|
||||
Then verify with `./dev_mysql_connect.sh`
|
||||
|
||||
#### Fixing dev MySql on Ubuntu 20.04
|
||||
|
||||
For some reason, the MySql default install on 20.04 gave me a bunch of trouble.
|
||||
@@ -0,0 +1,235 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
CREATE TABLE users (
|
||||
-- Not used for anything except replication.
|
||||
id BIGINT(20) NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- Visible "primary key"
|
||||
token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- Username is a lookup key; display_name allows the user to add custom case.
|
||||
username VARCHAR(20) NOT NULL,
|
||||
display_name VARCHAR(20) NOT NULL,
|
||||
|
||||
email_address VARCHAR(255) NOT NULL,
|
||||
email_confirmed BOOLEAN NOT NULL DEFAULT false,
|
||||
|
||||
-- The role assigned to the user confers permissions.
|
||||
user_role_slug VARCHAR(16) NOT NULL,
|
||||
|
||||
-- ========== CREDENTIALS ==========
|
||||
|
||||
-- Bcrypt password hash. Granted, there are newer methods:
|
||||
-- https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html
|
||||
-- Passwords may be a max of 64 characters per BCrypt.
|
||||
password_hash BINARY(60) NOT NULL,
|
||||
|
||||
-- Incremented with every update to the password.
|
||||
password_version INT NOT NULL DEFAULT 0,
|
||||
|
||||
-- ========== ABUSE TRACKING ==========
|
||||
|
||||
-- For abuse tracking.
|
||||
-- Wide enough for IPv4/6
|
||||
ip_address_creation VARCHAR(40) NOT NULL,
|
||||
ip_address_last_login VARCHAR(40) NOT NULL,
|
||||
ip_address_last_update VARCHAR(40) NOT NULL,
|
||||
|
||||
-- ========== DISPLAY / PROFILE ==========
|
||||
|
||||
-- Gravatar image hashes are precomputed.
|
||||
email_gravatar_hash CHAR(32) NOT NULL,
|
||||
|
||||
-- The profile of the user in markdown (user editable).
|
||||
profile_markdown TEXT NOT NULL,
|
||||
|
||||
-- Generated HTML (not user-editable).
|
||||
profile_rendered_html TEXT NOT NULL,
|
||||
|
||||
-- An uploaded avatar. Public hash in our bucket.
|
||||
-- avatar_public_bucket_hash CHAR(32) DEFAULT NULL,
|
||||
|
||||
-- ========== SOCIAL MEDIA ==========
|
||||
|
||||
-- Social media usernames
|
||||
-- These are not confirmed. We'll need to build an OAuth system to handle that.
|
||||
discord_username VARCHAR(36) DEFAULT NULL,
|
||||
twitter_username VARCHAR(36) DEFAULT NULL,
|
||||
twitch_username VARCHAR(36) DEFAULT NULL,
|
||||
patreon_username VARCHAR(36) DEFAULT NULL,
|
||||
github_username VARCHAR(36) DEFAULT NULL,
|
||||
cashapp_username VARCHAR(36) DEFAULT NULL,
|
||||
website_url VARCHAR(255) DEFAULT NULL,
|
||||
|
||||
-- ========== USER PREFERENCES ==========
|
||||
|
||||
-- If the user doesn't want to use gravatar and doesn't have an uploaded avatar.
|
||||
disable_gravatar BOOLEAN NOT NULL DEFAULT false,
|
||||
|
||||
-- (THIS MIGHT NOT BE USED)
|
||||
-- NB: DO NOT SORT!
|
||||
-- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
|
||||
-- Visibility preference used at TTS inference time (which can be changed on the result itself).
|
||||
-- Moderators will still see them.
|
||||
preferred_tts_result_visibility ENUM(
|
||||
'public',
|
||||
'hidden',
|
||||
'private'
|
||||
) NOT NULL DEFAULT 'public',
|
||||
|
||||
-- (THIS MIGHT NOT BE USED)
|
||||
-- NB: DO NOT SORT!
|
||||
-- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
|
||||
-- Visibility preference used at TTS inference time (which can be changed on the result itself).
|
||||
-- Moderators will still see them.
|
||||
preferred_w2l_result_visibility ENUM(
|
||||
'public',
|
||||
'hidden',
|
||||
'private'
|
||||
) NOT NULL DEFAULT 'public',
|
||||
|
||||
-- Auto play preferences
|
||||
-- Tri-state because we want to potentially change the default behavior.
|
||||
auto_play_audio_preference BOOLEAN DEFAULT NULL,
|
||||
auto_play_video_preference BOOLEAN DEFAULT NULL,
|
||||
|
||||
-- Favorite TTS voice to use by default
|
||||
-- maybe_preferred_tts_model_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- Favorite W2L model to use by default
|
||||
-- maybe_preferred_w2l_template_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- Settings
|
||||
-- DO NOT REORDER.
|
||||
-- dark_mode_preference ENUM(
|
||||
-- 'light-mode',
|
||||
-- 'dark-mode',
|
||||
-- 'use-clock'
|
||||
-- ) NOT NULL DEFAULT 'light-mode',
|
||||
|
||||
-- ========== STATS ==========
|
||||
|
||||
-- For tracking stats.
|
||||
-- The "cached" values are updated by a background job.
|
||||
-- cached_tts_rendered_counter INT(10) NOT NULL DEFAULT 0,
|
||||
-- cached_w2l_rendered_counter INT(10) NOT NULL DEFAULT 0,
|
||||
|
||||
-- ========== MODERATION DETAILS ==========
|
||||
|
||||
-- Different than deleted.
|
||||
-- Users still show up, but can't do anything.
|
||||
is_banned BOOLEAN NOT NULL DEFAULT false,
|
||||
|
||||
-- If a moderator has comments.
|
||||
maybe_mod_comments VARCHAR(255) DEFAULT NULL,
|
||||
|
||||
-- The last moderator that made changes.
|
||||
maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- ========== VECTOR CLOCK ==========
|
||||
|
||||
-- Incremented with every update.
|
||||
version INT NOT NULL DEFAULT 0,
|
||||
|
||||
-- ========== TIMESTAMPS ==========
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
-- If the user is deleted, we set this.
|
||||
-- This is different than banned. These users won't show up at all.
|
||||
user_deleted_at TIMESTAMP NULL,
|
||||
mod_deleted_at TIMESTAMP NULL,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY (token),
|
||||
UNIQUE KEY (username),
|
||||
UNIQUE KEY (email_address),
|
||||
KEY fk_user_role_slug (user_role_slug)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
|
||||
CREATE TABLE user_roles (
|
||||
-- Not used for anything except replication.
|
||||
id BIGINT(20) NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- Effective "primary key"
|
||||
slug CHAR(16) NOT NULL,
|
||||
|
||||
name VARCHAR(255) NOT NULL,
|
||||
|
||||
-- Usage
|
||||
can_use_tts BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
can_use_w2l BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
can_delete_own_tts_results BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
can_delete_own_w2l_results BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
can_delete_own_account BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
-- Contribution
|
||||
can_upload_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
can_upload_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
can_delete_own_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
can_delete_own_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
-- Moderation
|
||||
can_approve_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
-- Edit includes "lock_from_use" and "lock_from_modification"
|
||||
can_edit_other_users_profiles BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
can_edit_other_users_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
can_edit_other_users_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
can_delete_other_users_tts_models BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
can_delete_other_users_tts_results BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
can_delete_other_users_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
can_delete_other_users_w2l_results BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
can_ban_users BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
can_delete_users BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
-- Incremented with every update.
|
||||
version INT NOT NULL DEFAULT 0,
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY (slug)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
|
||||
CREATE TABLE user_sessions (
|
||||
-- Not used for anything except replication.
|
||||
id BIGINT(20) NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- Session entropy
|
||||
token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- Foreign key to user
|
||||
user_token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- Track each session's creation IP.
|
||||
-- Wide enough for IPv4/6
|
||||
ip_address_creation VARCHAR(40) NOT NULL,
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
-- Session termination time.
|
||||
-- This must be set by the server code, or the session is invalid
|
||||
expires_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
-- deletion = session termination
|
||||
-- Typically these are destroyed by users, but if in the future we allow mods to
|
||||
-- delete them, it doesn't really matter who did the deletion: sessions are not
|
||||
-- designed to be recoverable.
|
||||
deleted_at TIMESTAMP NULL,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY (token),
|
||||
KEY fk_user_token (user_token),
|
||||
KEY index_ip_address_creation (ip_address_creation)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
@@ -0,0 +1,110 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
CREATE TABLE generic_download_jobs (
|
||||
-- Not used for anything except replication.
|
||||
id BIGINT(20) NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- Effective "primary key" (PUBLIC)
|
||||
-- This is so the in-progress results can be looked up by the UI.
|
||||
token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- Idempotency token
|
||||
-- This is so the frontend client doesn't submit duplicate jobs.
|
||||
uuid_idempotency_token VARCHAR(36) NOT NULL,
|
||||
|
||||
-- ========== SUCCESS CASE ==========
|
||||
|
||||
-- If the job completes successfully, this is the result token.
|
||||
-- This is only populated on a successful result.
|
||||
on_success_downloaded_entity_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- The type of the object will vary based on the type of the upload,
|
||||
-- and we may include heuristics that auto-detect types in the future
|
||||
on_success_downloaded_entity_type VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- ========== UPLOAD DETAILS ==========
|
||||
|
||||
-- A "well defined" type of upload
|
||||
-- The first upload type will be "hifigan_model"
|
||||
download_type VARCHAR(32) NOT NULL,
|
||||
|
||||
-- Assume the same URL can be uploaded more than once (in case of errors or
|
||||
-- people sharing models)
|
||||
|
||||
download_url VARCHAR(512) NOT NULL,
|
||||
|
||||
-- The user-supplied title of the upload
|
||||
-- This might have different meanings for different upload types
|
||||
-- The "name" of the voice model, which might be complicated.
|
||||
-- If maybe_subject_token (etc.) is set, then it's authoritative instead.
|
||||
title VARCHAR(255) NOT NULL,
|
||||
|
||||
-- ========== CREATOR DETAILS AND PREFERENCES ==========
|
||||
|
||||
-- Foreign key to user
|
||||
creator_user_token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- For abuse tracking.
|
||||
-- Wide enough for IPv4/6
|
||||
creator_ip_address VARCHAR(40) NOT NULL,
|
||||
|
||||
-- The creator can set a desired visibility for their data.
|
||||
-- This does not always apply to every upload type.
|
||||
-- Additionally, some upload types may require moderator approval prior
|
||||
-- to being publicly listed, and this field has no bearing on that.
|
||||
-- NB: DO NOT CHANGE THE ORDER OF THE ENUM VALUES DURING SCHEMA MIGRATIONS.
|
||||
creator_set_visibility ENUM(
|
||||
'public',
|
||||
'hidden',
|
||||
'private'
|
||||
) NOT NULL DEFAULT 'public',
|
||||
|
||||
-- ========== JOB SYSTEM DETAILS ==========
|
||||
|
||||
-- Jobs begin as "pending", then transition to other states.
|
||||
--
|
||||
-- * Pending = job is ready to go
|
||||
-- * Started = job is running
|
||||
-- * Complete_Success = job is done (success)
|
||||
-- * Complete_Failure = job is done (failure)
|
||||
-- * Attempt_Failed = job failed but may retry.
|
||||
-- * Dead = job failed permanently.
|
||||
--
|
||||
-- Pending -> Started -> Complete_Success
|
||||
-- |-> Complete_Failure
|
||||
-- \-> Attempt_Failed -> Started -> { Complete, Failed, Dead }
|
||||
status ENUM(
|
||||
'pending',
|
||||
'started',
|
||||
'complete_success',
|
||||
'complete_failure',
|
||||
'attempt_failed',
|
||||
'dead') NOT NULL DEFAULT 'pending',
|
||||
|
||||
-- We can track this against a "max_attempt_count"
|
||||
attempt_count INT(3) NOT NULL DEFAULT 0,
|
||||
|
||||
-- If there is a failure, tell the user why.
|
||||
failure_reason VARCHAR(512) DEFAULT NULL,
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
-- Failed jobs will set a next attempt time.
|
||||
-- Subsequent tries can increase the timeout.
|
||||
-- Failures because of permissions require human intervention => [retry_at=null].
|
||||
-- Failures because of invalid files are dead => [status=dead].
|
||||
retry_at TIMESTAMP NULL,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY (token),
|
||||
UNIQUE KEY (uuid_idempotency_token),
|
||||
KEY fk_on_download_type (download_type),
|
||||
KEY fk_creator_user_token (creator_user_token),
|
||||
KEY index_status (status),
|
||||
KEY index_creator_ip_address (creator_ip_address)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
@@ -0,0 +1,128 @@
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
CREATE TABLE vocoder_models (
|
||||
-- Not used for anything except replication.
|
||||
id BIGINT(20) NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- Effective "primary key" (PUBLIC)
|
||||
token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- The architecture of the vocoder.
|
||||
-- Currently supported values: 'hifigan'
|
||||
vocoder_type VARCHAR(32) NOT NULL,
|
||||
|
||||
-- ========== MUTABLE METADATA ==========
|
||||
|
||||
-- The "name" of the vocoder, which might be complicated.
|
||||
title VARCHAR(255) NOT NULL,
|
||||
|
||||
-- The description of the model in markdown.
|
||||
description_markdown TEXT NOT NULL,
|
||||
|
||||
-- Generated HTML (not user-editable).
|
||||
description_rendered_html TEXT NOT NULL,
|
||||
|
||||
-- ========== MUTABLE MOD-ONLY METADATA ==========
|
||||
|
||||
-- Whether the vocoder is recommended for use
|
||||
is_staff_recommended BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
-- ========== IMMUTABLE PROVENANCE AND METADATA ==========
|
||||
|
||||
-- Where the file was originally downloaded (if it was downloaded)
|
||||
original_download_url VARCHAR(512) DEFAULT NULL,
|
||||
|
||||
-- The filename that was used at upload time (if available)
|
||||
original_filename VARCHAR(255) DEFAULT NULL,
|
||||
|
||||
-- File characteristics
|
||||
file_size_bytes INT(10) NOT NULL DEFAULT 0,
|
||||
|
||||
-- ========== CREATOR DETAILS ==========
|
||||
|
||||
-- The person that created the template.
|
||||
creator_user_token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- For abuse tracking.
|
||||
-- Wide enough for IPv4/6
|
||||
creator_ip_address_creation VARCHAR(40) NOT NULL,
|
||||
creator_ip_address_last_update VARCHAR(40) NOT NULL,
|
||||
|
||||
-- (THIS MIGHT NOT BE USED)
|
||||
-- NB: DO NOT SORT!
|
||||
-- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
|
||||
creator_set_visibility ENUM(
|
||||
'public',
|
||||
'hidden',
|
||||
'private'
|
||||
) NOT NULL DEFAULT 'public',
|
||||
|
||||
-- ========== BUCKET STORAGE ==========
|
||||
|
||||
-- The pytorch model
|
||||
-- For now, this will be a hash of the file contents.
|
||||
-- NB: NOT UNIQUE! We can allow duplicate uploads.
|
||||
private_bucket_hash CHAR(64) NOT NULL,
|
||||
|
||||
-- The "full url" of the bucket path, not including the hostname
|
||||
private_bucket_object_name VARCHAR(255) NOT NULL,
|
||||
|
||||
-- ========== MODERATION DETAILS ==========
|
||||
|
||||
-- Mods have locked the ability for non-authors (ie. people who didn't upload this)
|
||||
-- to assign the vocoder for use with their TTS models.
|
||||
-- TTS models that already incorporate the vocoder will not be affected.
|
||||
-- The vocoder will remain visible.
|
||||
-- This is independent of other visibility controls (is_mod_disabled_from_author_use,
|
||||
-- deleted_at, and creator_set_visibility)
|
||||
is_mod_disabled_from_public_use BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
-- Mods have locked the ability for the author (ie. the person that uploaded this)
|
||||
-- to assign the vocoder for use with their TTS models.
|
||||
-- TTS models that already incorporate the vocoder will not be affected.
|
||||
-- The vocoder will remain visible.
|
||||
-- This is independent of other visibility controls (is_mod_disabled_from_public_use,
|
||||
-- deleted_at, and creator_set_visibility)
|
||||
is_mod_disabled_from_author_use BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
-- Extremely popular models may be locked from deletion or modification by users.
|
||||
-- If true, only moderators can edit or delete the vocoder.
|
||||
is_mod_author_editing_locked BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
-- If a moderator has comments.
|
||||
maybe_mod_comments VARCHAR(255) DEFAULT NULL,
|
||||
|
||||
-- The last moderator that made changes.
|
||||
maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- ========== VECTOR CLOCK ==========
|
||||
|
||||
-- Incremented with every update.
|
||||
version INT NOT NULL DEFAULT 0,
|
||||
|
||||
-- ========== TIMESTAMPS ==========
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
-- If this is removed by a mod.
|
||||
-- It completely disappears from the system.
|
||||
user_deleted_at TIMESTAMP NULL,
|
||||
mod_deleted_at TIMESTAMP NULL,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY (token),
|
||||
KEY fk_creator_user_token (creator_user_token),
|
||||
KEY fk_maybe_mod_user_token (maybe_mod_user_token),
|
||||
KEY index_creator_ip_address_creation (creator_ip_address_creation),
|
||||
KEY index_creator_ip_address_last_update (creator_ip_address_last_update),
|
||||
KEY index_creator_set_visibility (creator_set_visibility),
|
||||
KEY index_private_bucket_hash (private_bucket_hash),
|
||||
KEY index_is_mod_disabled_from_public_use (is_mod_disabled_from_public_use),
|
||||
KEY index_is_mod_disabled_from_author_use (is_mod_disabled_from_author_use),
|
||||
KEY index_is_mod_author_editing_locked (is_mod_author_editing_locked)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
@@ -0,0 +1,124 @@
|
||||
-- NB: This is a manually squashed view of all the CREATE and ALTER statements,
|
||||
-- with comments attached to the fields for centralized documentation.
|
||||
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
CREATE TABLE tts_inference_jobs (
|
||||
id BIGINT(20) NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- Effective "primary key" (PUBLIC)
|
||||
-- This is so the in-progress results can be looked up by the UI.
|
||||
token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- Idempotency token from client
|
||||
-- This is so the frontend client doesn't submit duplicate jobs.
|
||||
uuid_idempotency_token VARCHAR(36) NOT NULL,
|
||||
|
||||
-- ========== SUCCESS CASE ==========
|
||||
|
||||
-- If the job completes successfully, this is the result token.
|
||||
-- This is only populated on a successful result.
|
||||
on_success_result_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- ========== INFERENCE DETAILS ==========
|
||||
|
||||
-- The model to use.
|
||||
-- This also determines which architecture we're using.
|
||||
model_token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- If set, use a different pretrained vocoder.
|
||||
maybe_override_pretrained_vocoder VARCHAR(64) DEFAULT NULL,
|
||||
|
||||
-- The raw, unprocessed user input.
|
||||
raw_inference_text TEXT NOT NULL,
|
||||
|
||||
-- ========== CREATOR DETAILS ==========
|
||||
|
||||
-- Foreign key to user
|
||||
-- If no user is logged in, this is null.
|
||||
maybe_creator_user_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- Based on a cookie sent by the frontend.
|
||||
-- We'll save this even if the user is logged in.
|
||||
maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- For abuse tracking.
|
||||
-- Wide enough for IPv4/6
|
||||
creator_ip_address VARCHAR(40) NOT NULL,
|
||||
|
||||
-- (THIS MIGHT NOT BE USED)
|
||||
-- NB: DO NOT SORT!
|
||||
-- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
|
||||
creator_set_visibility ENUM(
|
||||
'public',
|
||||
'hidden',
|
||||
'private'
|
||||
) NOT NULL DEFAULT 'public',
|
||||
|
||||
-- ========== SOURCE METADATA ==========
|
||||
|
||||
is_from_api BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
is_for_twitch BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
-- If true, the request gets routed to a special "debug" worker.
|
||||
is_debug_request BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
-- ========== JOB SYSTEM DETAILS ==========
|
||||
|
||||
-- Jobs begin as "pending", then transition to other states.
|
||||
--
|
||||
-- * Pending = job is ready to go
|
||||
-- * Started = job is running
|
||||
-- * Complete_Success = job is done (success)
|
||||
-- * Complete_Failure = job is done (failure)
|
||||
-- * Attempt_Failed = job failed but may retry.
|
||||
-- * Dead = job failed permanently.
|
||||
--
|
||||
-- Pending -> Started -> Complete_Success
|
||||
-- |-> Complete_Failure
|
||||
-- \-> Attempt_Failed -> Started -> { Complete, Failed, Dead }
|
||||
status ENUM(
|
||||
'pending',
|
||||
'started',
|
||||
'complete_success',
|
||||
'complete_failure',
|
||||
'attempt_failed',
|
||||
'dead') NOT NULL DEFAULT 'pending',
|
||||
|
||||
-- Priority *increases*, so a level of 2 will be higher than 1.
|
||||
-- These are the level semantics currently:
|
||||
-- - All jobs from anonymous FakeYou users have level 0.
|
||||
-- - All jobs from logged in FakeYou users have level 1.
|
||||
-- - All jobs from Twitch TTS (unpaid) have level 10 (ten).
|
||||
-- - (There will be future levels for paid Twitch and social FakeYou rewards.)
|
||||
priority_level TINYINT UNSIGNED NOT NULL DEFAULT 0,
|
||||
|
||||
-- We can track this against a "max_attempt_count"
|
||||
attempt_count INT(3) NOT NULL DEFAULT 0,
|
||||
|
||||
-- If there is a failure, tell the user why.
|
||||
failure_reason VARCHAR(512) DEFAULT NULL,
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
-- Failed jobs will set a next attempt time.
|
||||
-- Subsequent tries can increase the timeout.
|
||||
-- Failures because of permissions require human intervention => [retry_at=null].
|
||||
-- Failures because of invalid files are dead => [status=dead].
|
||||
retry_at TIMESTAMP NULL,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY (token),
|
||||
UNIQUE KEY (uuid_idempotency_token),
|
||||
KEY fk_on_success_result_token (on_success_result_token),
|
||||
KEY fk_model_token (model_token),
|
||||
KEY fk_maybe_creator_user_token (maybe_creator_user_token),
|
||||
KEY index_status (status),
|
||||
KEY index_creator_ip_address (creator_ip_address),
|
||||
KEY index_is_debug_request (is_debug_request)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
@@ -0,0 +1,217 @@
|
||||
-- NB: This is a manually squashed view of all the CREATE and ALTER statements,
|
||||
-- with comments attached to the fields for centralized documentation.
|
||||
|
||||
-- noinspection SqlDialectInspectionForFile
|
||||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
-- noinspection SqlResolveForFile
|
||||
|
||||
CREATE TABLE tts_models (
|
||||
-- Not used for anything except replication.
|
||||
id BIGINT(20) NOT NULL AUTO_INCREMENT,
|
||||
|
||||
-- Effective "primary key" (PUBLIC)
|
||||
token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- NB: DO NOT CHANGE ORDER; APPEND ONLY!
|
||||
-- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
|
||||
tts_model_type ENUM(
|
||||
'not-set',
|
||||
'tacotron2',
|
||||
'glowtts',
|
||||
'glowtts-vocodes',
|
||||
'talknet'
|
||||
) NOT NULL DEFAULT 'not-set',
|
||||
|
||||
-- The name of the type of pipeline to use, eg "legacy_fakeyou", "legacy_vocodes", "english_v1", etc.
|
||||
-- New pipelines may be added at any point in the future and the source of truth for them will mostly
|
||||
-- live in the `storyteller-ml` repository.
|
||||
-- Older models and brand new models will have this field set to null, and we'll infer which value to
|
||||
-- lazily backfill based on the creation date and language setting.
|
||||
text_pipeline_type VARCHAR(64) DEFAULT NULL,
|
||||
|
||||
-- For models distributed as archives/bundles (eg. TalkNet), we can pack
|
||||
-- in a vocoder. Note that this vocoder does not have to be used.
|
||||
has_self_contained_vocoder BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
-- For models distributed as archives/bundles (eg. TalkNet), we can pack
|
||||
-- in a duration model.
|
||||
has_self_contained_duration_model BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
-- For models distributed as archives/bundles (eg. TalkNet), we can pack
|
||||
-- in a pitch model.
|
||||
has_self_contained_pitch_model BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
-- NB(2022-07-05): THIS IS NOW MEANINGLESS AND SUPERSEDED BY `TEXT_PIPELINE_TYPE`.
|
||||
-- NB: DO NOT CHANGE ORDER; APPEND ONLY!
|
||||
-- How text should be handled.
|
||||
text_preprocessing_algorithm ENUM(
|
||||
'basic',
|
||||
'arpabet'
|
||||
) NOT NULL DEFAULT 'basic',
|
||||
|
||||
-- If set, use a different pretrained vocoder.
|
||||
-- If not set, use the website default. (Currently HifiGan-SS)
|
||||
maybe_default_pretrained_vocoder VARCHAR(64) DEFAULT NULL,
|
||||
|
||||
-- If set, we'll use a custom vocoder (from table vocoder_models) instead of a default vocoder.
|
||||
-- If not set, we'll use `maybe_default_pretrained_vocoder` (or the website default).
|
||||
maybe_custom_vocoder_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- Optional Pointer to a newer version of the voice
|
||||
-- If there's a newer version, we can disable this one.
|
||||
-- maybe_updated_model_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- We can set as a combination of ['username' + 'voice-name']
|
||||
-- There can be public aliases for voices, eg. a voice's default model.
|
||||
-- A user can change this.
|
||||
-- As such, these should not be foreign keys.
|
||||
-- updatable_slug VARCHAR(64) NOT NULL,
|
||||
|
||||
-- Can be linked to a well-known voice
|
||||
-- maybe_subject_token VARCHAR(32) DEFAULT NULL,
|
||||
-- maybe_actor_subject_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- ========== UNSTRUCTURED METADATA ==========
|
||||
|
||||
-- The full IETF BCP47 language tag (eg. en, en-US, es-419, ja-JP, pt, etc.)
|
||||
ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en',
|
||||
|
||||
-- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es")
|
||||
ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en',
|
||||
|
||||
-- The "name" of the voice model, which might be complicated.
|
||||
-- If maybe_subject_token (etc.) is set, then it's authoritative instead.
|
||||
title VARCHAR(255) NOT NULL,
|
||||
|
||||
-- The description of the model in markdown.
|
||||
description_markdown TEXT NOT NULL,
|
||||
|
||||
-- Generated HTML (not user-editable).
|
||||
description_rendered_html TEXT NOT NULL,
|
||||
|
||||
-- eg. Bill Clinton, Batman, or Goku.
|
||||
voice_name VARCHAR(255) DEFAULT NULL,
|
||||
|
||||
-- Optional; eg. Sean Schemmel
|
||||
actor_name VARCHAR(255) DEFAULT NULL,
|
||||
|
||||
-- Optional; eg. if the voice is "happy" or a singer "a-capella", etc.
|
||||
characteristic VARCHAR(255) DEFAULT NULL,
|
||||
|
||||
-- Optional; eg. if you train the model more than once.
|
||||
version_string VARCHAR(255) DEFAULT NULL,
|
||||
|
||||
-- Whether the voice features on the front of FakeYou or Storyteller Twitch TTS
|
||||
is_front_page_featured BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
is_twitch_featured BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
-- A suggested bot command prefix, eg. 'sonic' that could be interpreted as '/sonic'
|
||||
-- by a TTS system. These are unique strings that can only be applied to one model at
|
||||
-- a time at most. Only moderators can set this, and it's only for the best voices.
|
||||
maybe_suggested_unique_bot_command VARCHAR(255) DEFAULT NULL,
|
||||
|
||||
-- ========== IMMUTABLE PROVENANCE AND METADATA ==========
|
||||
|
||||
-- Where the file was originally downloaded (if it was downloaded)
|
||||
original_download_url VARCHAR(512) DEFAULT NULL,
|
||||
|
||||
-- The filename that was used at upload time (if available)
|
||||
original_filename VARCHAR(255) DEFAULT NULL,
|
||||
|
||||
-- File characteristics
|
||||
file_size_bytes INT(10) NOT NULL DEFAULT 0,
|
||||
|
||||
-- ========== CREATOR DETAILS ==========
|
||||
|
||||
-- The person that created the template.
|
||||
creator_user_token VARCHAR(32) NOT NULL,
|
||||
|
||||
-- For abuse tracking.
|
||||
-- Wide enough for IPv4/6
|
||||
creator_ip_address_creation VARCHAR(40) NOT NULL,
|
||||
creator_ip_address_last_update VARCHAR(40) NOT NULL,
|
||||
|
||||
-- (THIS MIGHT NOT BE USED)
|
||||
-- NB: DO NOT SORT!
|
||||
-- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
|
||||
creator_set_visibility ENUM(
|
||||
'public',
|
||||
'hidden',
|
||||
'private'
|
||||
) NOT NULL DEFAULT 'public',
|
||||
|
||||
-- ========== BUCKET STORAGE ==========
|
||||
|
||||
-- The pytorch model
|
||||
-- For now, this will be a hash of the file contents.
|
||||
-- NB: NOT UNIQUE! We can allow duplicate uploads.
|
||||
private_bucket_hash CHAR(64) NOT NULL,
|
||||
-- The "full url" version of the path
|
||||
private_bucket_object_name VARCHAR(255) NOT NULL,
|
||||
|
||||
-- Whether the file is a zip archive (rather than a model file).
|
||||
private_bucket_object_is_archive BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
-- ========== RATING AND RANKING ==========
|
||||
|
||||
-- Calculated average, on a scale of 0-100
|
||||
-- Null with zero ratings.
|
||||
-- calculated_average_score INT(3) DEFAULT NULL,
|
||||
|
||||
-- Other metrics
|
||||
-- calculated_total_ratings_submitted_count INT(10) NOT NULL DEFAULT 0,
|
||||
-- calculated_total_uses_count BIGINT(10) NOT NULL DEFAULT 0,
|
||||
|
||||
-- ========== MODERATION DETAILS ==========
|
||||
|
||||
-- Mods have to approve of tts models for them to show up in a public index.
|
||||
-- Models can be used by the author (or other parties that know the URL) before
|
||||
-- they're approved, but unapproved models won't show up in public indices.
|
||||
-- This is independent of other visibility controls (is_mod_disabled, deleted_at,
|
||||
-- and creator_set_visibility)
|
||||
is_public_listing_approved BOOLEAN DEFAULT NULL,
|
||||
|
||||
-- Extremely popular models may be locked from deletion or modification by users.
|
||||
is_locked_from_user_modification BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
-- In this case, a moderator disables it from being used for inference.
|
||||
-- This also disables it for the creator.
|
||||
-- Unlike deletion, it remains "visible" to those that have access.
|
||||
is_locked_from_use BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
-- If a moderator has comments.
|
||||
maybe_mod_comments VARCHAR(255) DEFAULT NULL,
|
||||
|
||||
-- The last moderator that made changes.
|
||||
maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
|
||||
|
||||
-- ========== VECTOR CLOCK ==========
|
||||
|
||||
-- Incremented with every update.
|
||||
version INT NOT NULL DEFAULT 0,
|
||||
|
||||
-- ========== TIMESTAMPS ==========
|
||||
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
-- If this is removed by a mod.
|
||||
-- It completely disappears from the system.
|
||||
user_deleted_at TIMESTAMP NULL,
|
||||
mod_deleted_at TIMESTAMP NULL,
|
||||
|
||||
-- INDICES --
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY (token),
|
||||
KEY fk_creator_user_token (creator_user_token),
|
||||
KEY fk_maybe_mod_user_token (maybe_mod_user_token),
|
||||
KEY fk_maybe_custom_vocoder_token (maybe_custom_vocoder_token),
|
||||
KEY index_creator_ip_address_creation (creator_ip_address_creation),
|
||||
KEY index_creator_ip_address_last_update (creator_ip_address_last_update),
|
||||
KEY index_creator_set_visibility (creator_set_visibility),
|
||||
KEY index_private_bucket_hash (private_bucket_hash),
|
||||
KEY index_is_public_listing_approved (is_public_listing_approved),
|
||||
KEY index_is_locked_from_user_modification (is_locked_from_user_modification),
|
||||
KEY index_is_locked_from_use (is_locked_from_use)
|
||||
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user