Change the save button disabled logic

.github/workflows/artcraft-macos-publish.yml

@@ -1,232 +0,0 @@
-name: "Artcraft MacOS Publish"
-
-on:
-  push:
-    branches:
-      - release
-      - artcraft-release
-      - artcraft-macos  # Additional branch for MacOS testing
-
-# This workflow creates Universal macOS builds with signing certificate support.
-# On each push to the release branches it will create or update a GitHub release, build your app, and upload the artifacts to the release.
-
-jobs:
-  publish-artcraft:
-    permissions:
-      contents: write
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # Universal macOS builds (arm64 and x86_64)
-          - platform: 'macos-latest-xlarge'
-            args: '--target universal-apple-darwin'
-          # Uncomment below for additional platforms
-          # - platform: 'ubuntu-22.04'
-          #   args: ''
-          # - platform: 'windows-latest'
-          #   args: ''
-
-    runs-on: ${{ matrix.platform }}
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: setup node
-        uses: actions/setup-node@v4
-        with:
-          node-version: lts/*
-
-      - name: install Rust stable
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          targets: ${{ matrix.platform == 'macos-latest-xlarge' && 'aarch64-apple-darwin,x86_64-apple-darwin' || '' }}
-
-      - name: install dependencies (ubuntu only)
-        if: matrix.platform == 'ubuntu-22.04'
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf
-
-      - name: install frontend dependencies
-        run: |
-          # Navigate to the frontend directory to install dependencies
-          cd ./frontend
-
-          npm cache clean --force
-
-          echo "Installing all workspace dependencies..."
-          npm ci
-
-          echo "=== Version Information ==="
-          echo "Node version:"
-          node --version
-          echo "NPM version:"
-          npm --version
-
-          # Remove mismatched @nx packages and reinstall with correct versions
-          npm install --save-dev @nx/remix@21.1.1 @nx/js@21.1.1 @nx/react@21.1.1 @nx/vite@21.1.1 @nx/eslint@21.1.1
-
-          echo "=== Nx versions after adjustment ==="
-          npm list --depth=0 | grep nx
-
-      - name: import Apple Developer Certificate
-        if: matrix.platform == 'macos-latest-xlarge'
-        env:
-          APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
-          APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
-          KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
-        run: |
-          echo "Starting certificate import process..."
-
-          # Create certificate file
-          echo $APPLE_CERTIFICATE | base64 --decode > certificate.p12
-
-          # Debug: Check if certificate file exists and has content
-          echo "Certificate file info:"
-          ls -la certificate.p12
-          file certificate.p12
-
-          # Debug: Check certificate contents without password (just structure)
-          echo "Certificate structure (may show error if password protected):"
-          openssl pkcs12 -info -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -noout || echo "Could not read certificate structure"
-
-          # Debug: Try to list what's in the p12 file
-          echo "Attempting to list certificate contents:"
-          security cms -D -i certificate.p12 || echo "Not a CMS file, trying PKCS12..."
-          openssl pkcs12 -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -info -nodes 2>&1 | head -20 || echo "Failed to read PKCS12 contents"
-
-          # Debug: Extract certificate details to verify type and expiration
-          echo "Extracting certificate details..."
-          # Extract just the certificate portion and examine it
-          openssl pkcs12 -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -nokeys 2>/dev/null | openssl x509 -text -noout | grep -E "(Subject:|Issuer:|Not Before|Not After)" || echo "Failed to parse certificate details"
-
-          # Also try to see what type of certificate this is
-          echo "Certificate type check:"
-          openssl pkcs12 -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -nokeys 2>/dev/null | openssl x509 -text -noout | grep -E "(CN=|OU=)" || echo "Could not determine certificate type"
-
-          # Debug: Check if private key exists
-          echo "Checking for private key..."
-          openssl pkcs12 -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -nocerts -nodes 2>&1 | grep -E "(PRIVATE KEY|RSA KEY)" | head -5 || echo "No private key found"
-
-          # Create temporary keychain
-          security create-keychain -p "$KEYCHAIN_PASSWORD" build.keychain
-          security default-keychain -s build.keychain
-          security unlock-keychain -p "$KEYCHAIN_PASSWORD" build.keychain
-          security set-keychain-settings -t 3600 -u build.keychain
-
-          # Import certificate - ensure both certificate and private key are imported
-          echo "Importing certificate into keychain..."
-          # Import the p12 directly - this is the most reliable method
-          security import certificate.p12 -k build.keychain -P "$APPLE_CERTIFICATE_PASSWORD" -T /usr/bin/codesign -T /usr/bin/security -T /usr/bin/productbuild
-
-          # Alternative: Try with additional flags if the first import didn't create an identity
-          echo "Verifying import..."
-          if ! security find-identity -v build.keychain | grep -q "1 valid identities found"; then
-            echo "First import didn't create identity, trying with -A flag..."
-            security import certificate.p12 -k build.keychain -P "$APPLE_CERTIFICATE_PASSWORD" -A
-          fi
-
-          # Set trust settings for the certificate
-          echo "Setting trust settings..."
-          security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$KEYCHAIN_PASSWORD" build.keychain
-
-          # Unlock the keychain again to ensure it's accessible
-          security unlock-keychain -p "$KEYCHAIN_PASSWORD" build.keychain
-
-          # Add the keychain to the search list
-          security list-keychains -d user -s build.keychain $(security list-keychains -d user | sed 's/^[[:space:]]*"//;s/"$//')
-
-          # List all certificates to debug
-          echo "Listing all keychains:"
-          security list-keychains
-
-          echo "Listing all certificates in build keychain:"
-          security find-certificate -a -p build.keychain || echo "No certificates found"
-
-          echo "Detailed certificate info:"
-          security find-certificate -c "Learning Machines" build.keychain || echo "Certificate 'Learning Machines' not found"
-
-          # Check certificate trust settings
-          echo "Checking certificate trust settings:"
-          security dump-trust-settings || echo "No custom trust settings"
-
-          # Try to find codesigning identities
-          echo "Finding all identities in keychain:"
-          security find-identity -v build.keychain || echo "No identities found"
-
-          echo "Finding codesigning identities:"
-          security find-identity -v -p codesigning build.keychain || echo "No codesigning identities found"
-
-          # Additional debug: Check if this is a valid Developer ID cert
-          echo "Checking for Developer ID certificates:"
-          security find-certificate -a build.keychain | grep -A 5 -B 5 "Developer ID" || echo "No Developer ID certificates found"
-
-      - name: verify certificate
-        if: matrix.platform == 'macos-latest-xlarge'
-        run: |
-          echo "=== Certificate Verification ==="
-
-          echo "1. Checking codesigning identities:"
-          security find-identity -v -p codesigning build.keychain || echo "No codesigning identities found"
-
-          echo -e "\n2. Checking all identities:"
-          security find-identity -v build.keychain || echo "No identities found"
-
-          echo -e "\n3. Checking certificates in keychain:"
-          security find-certificate -c "Learning Machines" build.keychain -p | openssl x509 -text -noout | grep -E "(Subject:|CN=)" || echo "Could not parse certificate"
-
-          # Set the correct signing identity
-          echo "APPLE_SIGNING_IDENTITY=Developer ID Application: Learning Machines LLC (DJ6XS33FX8)" >> $GITHUB_ENV
-          echo "Proceeding with identity: Developer ID Application: Learning Machines LLC (DJ6XS33FX8)"
-
-          # Also set the team ID
-          echo "APPLE_DEVELOPMENT_TEAM=DJ6XS33FX8" >> $GITHUB_ENV
-
-      - name: Pre-build frontend to avoid Tauri build issues
-        run: |
-          # Build the frontend app from the correct directory
-          cd ./frontend
-
-          # Increase Node.js memory limit for the build
-          export NODE_OPTIONS="--max-old-space-size=8192"
-
-          # Use npx to ensure we use the local nx version
-          echo "Building with npx nx..."
-          npx nx run editor3d:build
-
-          echo "Checking build output..."
-          ls -la apps/editor3d/dist/ | head -10
-
-          # Update Tauri config to skip the build (since we already built)
-          cd ${{ github.workspace }}/crates/desktop/artcraft
-          cp tauri.conf.json tauri.conf.json.bak
-          # Set beforeBuildCommand to empty or a no-op command
-          if command -v jq &> /dev/null; then
-            jq '.build.beforeBuildCommand = ""' tauri.conf.json > tauri.conf.json.tmp
-            mv tauri.conf.json.tmp tauri.conf.json
-          else
-            node -e "const fs = require('fs'); const config = JSON.parse(fs.readFileSync('tauri.conf.json', 'utf8')); config.build.beforeBuildCommand = ''; fs.writeFileSync('tauri.conf.json', JSON.stringify(config, null, 2));"
-          fi
-
-
-      - name: build and publish
-        uses: tauri-apps/tauri-action@v0
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          APPLE_ID: ${{ secrets.APPLE_ID }}
-          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
-          APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
-          APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
-          APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
-          APPLE_SIGNING_IDENTITY: ${{ env.APPLE_SIGNING_IDENTITY }}
-          APPLE_DEVELOPMENT_TEAM: ${{ env.APPLE_DEVELOPMENT_TEAM }}
-          # Use the SQLx query cache for SQLite
-          SQLX_OFFLINE: true
-        with:
-          projectPath: ./crates/desktop/artcraft
-          tagName: artcraft-v__VERSION__ # the action automatically replaces __VERSION__ with the app version.
-          releaseName: "Artcraft v__VERSION__"
-          releaseBody: "See the assets to download this version and install."
-          releaseDraft: true
-          prerelease: false
-          args: ${{ matrix.args }}

.github/workflows/artcraft-windows-publish.yml

@@ -1,96 +0,0 @@
-name: "Artcraft Windows Publish"
-
-on:
-  push:
-    branches:
-      - release
-      - artcraft-release
-      - artcraft-windows  # Additional branch for Windows testing
-
-# This workflow creates Windows builds.
-# On each push to the release branches it will create or update a GitHub release, build your app, and upload the artifacts to the release.
-
-jobs:
-  publish-artcraft-windows:
-    permissions:
-      contents: write
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # Windows builds
-          - platform: 'windows-latest'
-            args: ''
-
-    runs-on: ${{ matrix.platform }}
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: setup node
-        uses: actions/setup-node@v4
-        with:
-          node-version: lts/*
-
-      - name: install Rust stable
-        uses: dtolnay/rust-toolchain@stable
-
-      - name: install frontend dependencies
-        run: |
-          # Navigate to the frontend directory to install dependencies
-          cd ./frontend
-
-          npm cache clean --force
-
-          echo "Installing all workspace dependencies..."
-          npm ci
-
-          echo "=== Version Information ==="
-          echo "Node version:"
-          node --version
-          echo "NPM version:"
-          npm --version
-
-          # Remove mismatched @nx packages and reinstall with correct versions
-          npm install --save-dev @nx/remix@21.1.1 @nx/js@21.1.1 @nx/react@21.1.1 @nx/vite@21.1.1 @nx/eslint@21.1.1
-
-          echo "=== Nx versions after adjustment ==="
-          npm list --depth=0 | findstr nx
-
-      - name: Pre-build frontend to avoid Tauri build issues
-        run: |
-          # Build the frontend app from the correct directory
-          cd ./frontend
-
-          # Increase Node.js memory limit for the build
-          $env:NODE_OPTIONS="--max-old-space-size=8192"
-
-          # Use npx to ensure we use the local nx version
-          echo "Building with npx nx..."
-          npx nx run editor3d:build
-
-          echo "Checking build output..."
-          dir apps/editor3d/dist/ | Select-Object -First 10
-
-          # Update Tauri config to skip the build (since we already built)
-          cd ${{ github.workspace }}/crates/desktop/artcraft
-          Copy-Item tauri.conf.json tauri.conf.json.bak
-
-          # Set beforeBuildCommand to empty using PowerShell
-          $config = Get-Content tauri.conf.json | ConvertFrom-Json
-          $config.build.beforeBuildCommand = ""
-          $config | ConvertTo-Json -Depth 10 | Set-Content tauri.conf.json
-
-      - name: build and publish
-        uses: tauri-apps/tauri-action@v0
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          # Use the SQLx query cache for SQLite
-          SQLX_OFFLINE: true
-        with:
-          projectPath: ./crates/desktop/artcraft
-          tagName: artcraft-windows-v__VERSION__ # the action automatically replaces __VERSION__ with the app version.
-          releaseName: "Artcraft Windows v__VERSION__"
-          releaseBody: "Windows build - See the assets to download this version and install."
-          releaseDraft: true
-          prerelease: false
-          args: ${{ matrix.args }}

.gitignore

@@ -24,13 +24,11 @@ twitch_secrets.toml
 *-secrets.env
 
 # Utilities with secrets
+script/production_database_secret_backup.sh
+script/production_database_secret_connect.sh
 production_database_secret_connect.sh
 production_database_secret_query.sh
 production_database_voice_clone_report_download.sh
-script/artcraft/mac_signing_build.sh
-script/production_database_secret_backup.sh
-script/production_database_secret_connect.sh
-script/socat_dev_database_connect.sh
 
 # Scratch data
 scratch/

.sqlx/query-002cdb3d086f3b4181e2997befa54bd8516cb1d3d3bb6e389862695318d5c299.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "MySQL",
-  "query": "\nINSERT INTO generic_inference_jobs\nSET\n  token = ?,\n  uuid_idempotency_token = ?,\n\n  job_type = ?,\n\n  maybe_external_third_party = ?,\n  maybe_external_third_party_id = ?,\n\n  product_category = ?,\n  inference_category = ?,\n\n  maybe_model_type = NULL,\n  maybe_model_token = NULL,\n\n  maybe_input_source_token = NULL,\n  maybe_input_source_token_type = NULL,\n\n  maybe_download_url = NULL,\n  maybe_cover_image_media_file_token = NULL,\n  \n  maybe_prompt_token = ?,\n\n  maybe_raw_inference_text = NULL,\n\n  maybe_inference_args = ?,\n\n  maybe_creator_user_token = ?,\n  maybe_creator_anonymous_visitor_token = ?,\n  creator_ip_address = ?,\n  creator_set_visibility = ?,\n\n  priority_level = 0,\n  is_keepalive_required = FALSE,\n  max_duration_seconds = 0,\n\n  is_debug_request = FALSE,\n  maybe_routing_tag = NULL,\n\n  status = ?\n        ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 14
-    },
-    "nullable": []
-  },
-  "hash": "002cdb3d086f3b4181e2997befa54bd8516cb1d3d3bb6e389862695318d5c299"
-}

.sqlx/query-01a0bf33e8331c19dc73f064c54eaa863bec158ef7713450a06c75e246f5f08c.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "\n    INSERT INTO tasks (\n      id,\n      task_status,\n      task_type,\n      model_type,\n      provider,\n      provider_job_id,\n      frontend_caller,\n      frontend_subscriber_id,\n      frontend_subscriber_payload\n    )\n    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)\n  ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 9
-    },
-    "nullable": []
-  },
-  "hash": "01a0bf33e8331c19dc73f064c54eaa863bec158ef7713450a06c75e246f5f08c"
-}

.sqlx/query-08f52f15acf6afc7091a525e3dc88f706c71c3579e5e81aa9903807d85f469ef.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "MySQL",
-  "query": "\nINSERT INTO prompts\nSET\n  token = ?,\n  prompt_type = ?,\n\n  maybe_creator_user_token = ?,\n  \n  maybe_model_type = ?,\n  maybe_generation_provider = ?,\n\n  maybe_positive_prompt = ?,\n  maybe_negative_prompt = ?,\n  \n  maybe_other_args = ?,\n\n  creator_ip_address = ?\n        ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 9
-    },
-    "nullable": []
-  },
-  "hash": "08f52f15acf6afc7091a525e3dc88f706c71c3579e5e81aa9903807d85f469ef"
-}

.sqlx/query-094eb099cde85835db0662c2700c6b82644b51bf9b3609391ed4d2945fe5e98f.json

@@ -1,69 +0,0 @@
-{
-  "db_name": "MySQL",
-  "query": "\nSELECT\n    pci.media_token as `media_token: tokens::tokens::media_files::MediaFileToken`,\n    pci.context_semantic_type as `context_semantic_type: enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType`,\n\n    m.public_bucket_directory_hash,\n    m.maybe_public_bucket_prefix,\n    m.maybe_public_bucket_extension\n\nFROM prompt_context_items pci\nJOIN media_files m\nON\n    pci.media_token = m.token\nWHERE\n    pci.prompt_token = ?\nORDER BY pci.id ASC\n        ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "media_token: tokens::tokens::media_files::MediaFileToken",
-        "type_info": {
-          "type": "VarString",
-          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
-          "char_set": 224,
-          "max_size": 128
-        }
-      },
-      {
-        "ordinal": 1,
-        "name": "context_semantic_type: enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType",
-        "type_info": {
-          "type": "VarString",
-          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
-          "char_set": 224,
-          "max_size": 64
-        }
-      },
-      {
-        "ordinal": 2,
-        "name": "public_bucket_directory_hash",
-        "type_info": {
-          "type": "VarString",
-          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
-          "char_set": 224,
-          "max_size": 128
-        }
-      },
-      {
-        "ordinal": 3,
-        "name": "maybe_public_bucket_prefix",
-        "type_info": {
-          "type": "VarString",
-          "flags": "BINARY",
-          "char_set": 224,
-          "max_size": 64
-        }
-      },
-      {
-        "ordinal": 4,
-        "name": "maybe_public_bucket_extension",
-        "type_info": {
-          "type": "VarString",
-          "flags": "BINARY",
-          "char_set": 224,
-          "max_size": 64
-        }
-      }
-    ],
-    "parameters": {
-      "Right": 1
-    },
-    "nullable": [
-      false,
-      false,
-      false,
-      true,
-      true
-    ]
-  },
-  "hash": "094eb099cde85835db0662c2700c6b82644b51bf9b3609391ed4d2945fe5e98f"
-}

.sqlx/query-13ca06bcbb4a576f8a2229a8554b7488d379baa381d8d5cb60972750e81b2173.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\nINSERT INTO prompts\nSET\n  token = ?,\n  prompt_type = ?,\n\n  maybe_creator_user_token = ?,\n\n  maybe_positive_prompt = ?,\n  maybe_negative_prompt = ?,\n  maybe_other_args = ?,\n\n  creator_ip_address = ?\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 7
+    },
+    "nullable": []
+  },
+  "hash": "13ca06bcbb4a576f8a2229a8554b7488d379baa381d8d5cb60972750e81b2173"
+}

.sqlx/query-19708db99f2b3c6283d2115a511e0eb1c99fab396ad0effd9eb2dd79926e9c93.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "MySQL",
-  "query": "\nUPDATE generic_inference_jobs\nSET\n  status = 'cancelled_by_system',\n  retry_at = NULL,\n  internal_debugging_failure_reason = 'reap_stale_fakeyou_jobs'\nWHERE\n  job_type != 'fal_queue'\n  AND status IN (\n    'pending',\n    'started',\n    'attempt_failed'\n   )\n   AND created_at > NOW() - INTERVAL 30 MINUTE\nLIMIT 5000\n        ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 0
-    },
-    "nullable": []
-  },
-  "hash": "19708db99f2b3c6283d2115a511e0eb1c99fab396ad0effd9eb2dd79926e9c93"
-}

.sqlx/query-61744bbf83f7cf00348bcc21e868d9c506fec46da4a33ac50d63c6d9e00c0cec.json

@@ -0,0 +1,102 @@
+{
+  "db_name": "MySQL",
+  "query": "\nSELECT\n    p.token as `token: tokens::tokens::prompts::PromptToken`,\n\n    p.prompt_type as `prompt_type: enums::by_table::prompts::prompt_type::PromptType`,\n\n    p.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n\n    p.maybe_positive_prompt,\n    p.maybe_negative_prompt,\n    p.maybe_other_args,\n\n    p.creator_ip_address,\n    p.created_at\n\nFROM prompts as p\nWHERE\n    p.token = ?\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "token: tokens::tokens::prompts::PromptToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 1,
+        "name": "prompt_type: enums::by_table::prompts::prompt_type::PromptType",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 64
+        }
+      },
+      {
+        "ordinal": 2,
+        "name": "maybe_creator_user_token: tokens::tokens::users::UserToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "MULTIPLE_KEY | BINARY",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 3,
+        "name": "maybe_positive_prompt",
+        "type_info": {
+          "type": "Blob",
+          "flags": "BLOB | BINARY",
+          "char_set": 224,
+          "max_size": 262140
+        }
+      },
+      {
+        "ordinal": 4,
+        "name": "maybe_negative_prompt",
+        "type_info": {
+          "type": "Blob",
+          "flags": "BLOB | BINARY",
+          "char_set": 224,
+          "max_size": 262140
+        }
+      },
+      {
+        "ordinal": 5,
+        "name": "maybe_other_args",
+        "type_info": {
+          "type": "Blob",
+          "flags": "BLOB | BINARY",
+          "char_set": 224,
+          "max_size": 262140
+        }
+      },
+      {
+        "ordinal": 6,
+        "name": "creator_ip_address",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 160
+        }
+      },
+      {
+        "ordinal": 7,
+        "name": "created_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP",
+          "char_set": 63,
+          "max_size": 19
+        }
+      }
+    ],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": [
+      false,
+      false,
+      true,
+      true,
+      true,
+      true,
+      false,
+      false
+    ]
+  },
+  "hash": "61744bbf83f7cf00348bcc21e868d9c506fec46da4a33ac50d63c6d9e00c0cec"
+}

.sqlx/query-90366e1705a0343bdfe4fc472d701b8e1dbd3a2dabedaadb633ac3542de72fa7.json

@@ -48,7 +48,7 @@
         "name": "maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken",
         "type_info": {
           "type": "VarString",
-          "flags": "MULTIPLE_KEY | BINARY",
+          "flags": "BINARY",
           "char_set": 224,
           "max_size": 128
         }

.sqlx/query-b4470e607c005421dd8504b9f1d695080a581936ac6ce00b6f437b2764e741be.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "MySQL",
-  "query": "\n        INSERT INTO media_files\n        SET\n            token = ?,\n\n            media_class = ?,\n            media_type = ?,\n\n            is_user_upload = ?,\n            is_intermediate_system_file = ?,\n\n            origin_category = ?, \n            origin_product_category = ?, \n            maybe_origin_model_type = ?, \n            maybe_origin_model_token = ?, \n            maybe_origin_filename = ?,\n\n            is_batch_generated = ?,\n            maybe_batch_token = ?,\n\n            maybe_mime_type = ?,\n            file_size_bytes = ?,\n            maybe_duration_millis = ?,\n            maybe_audio_encoding = ?, \n            maybe_video_encoding = ?, \n            maybe_frame_width = ?, \n            maybe_frame_height = ?,\n            maybe_prompt_token = ?,\n            checksum_sha2 = ?,\n            \n            maybe_engine_category = ?,\n\n            maybe_title = ?,\n            maybe_text_transcript = ?,\n\n            maybe_scene_source_media_file_token = ?,\n\n            public_bucket_directory_hash = ?, \n            maybe_public_bucket_prefix = ?, \n            maybe_public_bucket_extension = ?, \n\n            maybe_creator_user_token = ?, \n            maybe_creator_anonymous_visitor_token = ?, \n\n            creator_ip_address = ?, \n            creator_set_visibility = ?, \n\n            maybe_creator_file_synthetic_id = ?, \n            maybe_creator_category_synthetic_id = ?,\n\n            extra_file_modification_info = ?,\n\n            maybe_mod_user_token = ?, \n            is_generated_on_prem = ?, \n            generated_by_worker = ?, \n            generated_by_cluster = ?\n        ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 39
-    },
-    "nullable": []
-  },
-  "hash": "b4470e607c005421dd8504b9f1d695080a581936ac6ce00b6f437b2764e741be"
-}

.sqlx/query-b46b0d022790f4e10f6e47126df0fdc55172bf8cda77d4e0b2afac9e6ebd1630.json

@@ -1,124 +0,0 @@
-{
-  "db_name": "MySQL",
-  "query": "\nSELECT\n    p.token as `token: tokens::tokens::prompts::PromptToken`,\n\n    p.prompt_type as `prompt_type: enums::by_table::prompts::prompt_type::PromptType`,\n\n    p.maybe_model_type as `maybe_model_type: enums::common::model_type::ModelType`,\n    p.maybe_generation_provider as `maybe_generation_provider: enums::common::generation_provider::GenerationProvider`,\n\n    p.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n\n    p.maybe_positive_prompt,\n    p.maybe_negative_prompt,\n    p.maybe_other_args,\n\n    p.creator_ip_address,\n    p.created_at\n\nFROM prompts as p\nWHERE\n    p.token = ?\n        ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "token: tokens::tokens::prompts::PromptToken",
-        "type_info": {
-          "type": "VarString",
-          "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
-          "char_set": 224,
-          "max_size": 128
-        }
-      },
-      {
-        "ordinal": 1,
-        "name": "prompt_type: enums::by_table::prompts::prompt_type::PromptType",
-        "type_info": {
-          "type": "VarString",
-          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
-          "char_set": 224,
-          "max_size": 64
-        }
-      },
-      {
-        "ordinal": 2,
-        "name": "maybe_model_type: enums::common::model_type::ModelType",
-        "type_info": {
-          "type": "VarString",
-          "flags": "BINARY",
-          "char_set": 224,
-          "max_size": 128
-        }
-      },
-      {
-        "ordinal": 3,
-        "name": "maybe_generation_provider: enums::common::generation_provider::GenerationProvider",
-        "type_info": {
-          "type": "VarString",
-          "flags": "BINARY",
-          "char_set": 224,
-          "max_size": 64
-        }
-      },
-      {
-        "ordinal": 4,
-        "name": "maybe_creator_user_token: tokens::tokens::users::UserToken",
-        "type_info": {
-          "type": "VarString",
-          "flags": "MULTIPLE_KEY | BINARY",
-          "char_set": 224,
-          "max_size": 128
-        }
-      },
-      {
-        "ordinal": 5,
-        "name": "maybe_positive_prompt",
-        "type_info": {
-          "type": "Blob",
-          "flags": "BLOB | BINARY",
-          "char_set": 224,
-          "max_size": 262140
-        }
-      },
-      {
-        "ordinal": 6,
-        "name": "maybe_negative_prompt",
-        "type_info": {
-          "type": "Blob",
-          "flags": "BLOB | BINARY",
-          "char_set": 224,
-          "max_size": 262140
-        }
-      },
-      {
-        "ordinal": 7,
-        "name": "maybe_other_args",
-        "type_info": {
-          "type": "Blob",
-          "flags": "BLOB | BINARY",
-          "char_set": 224,
-          "max_size": 262140
-        }
-      },
-      {
-        "ordinal": 8,
-        "name": "creator_ip_address",
-        "type_info": {
-          "type": "VarString",
-          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
-          "char_set": 224,
-          "max_size": 160
-        }
-      },
-      {
-        "ordinal": 9,
-        "name": "created_at",
-        "type_info": {
-          "type": "Timestamp",
-          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP",
-          "char_set": 63,
-          "max_size": 19
-        }
-      }
-    ],
-    "parameters": {
-      "Right": 1
-    },
-    "nullable": [
-      false,
-      false,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      false,
-      false
-    ]
-  },
-  "hash": "b46b0d022790f4e10f6e47126df0fdc55172bf8cda77d4e0b2afac9e6ebd1630"
-}

.sqlx/query-c58f414b5e4d0f0420d1894e2f40311e5613919f49f3ac91f81f5fc2bef1bea8.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\n        INSERT INTO media_files\n        SET\n            token = ?,\n\n            media_class = ?,\n            media_type = ?,\n\n            is_user_upload = ?,\n            is_intermediate_system_file = ?,\n\n            origin_category = ?, \n            origin_product_category = ?, \n            maybe_origin_model_type = ?, \n            maybe_origin_model_token = ?, \n            maybe_origin_filename = ?,\n\n            is_batch_generated = ?,\n            maybe_batch_token = ?,\n\n            maybe_mime_type = ?,\n            file_size_bytes = ?,\n            maybe_duration_millis = ?,\n            maybe_audio_encoding = ?, \n            maybe_video_encoding = ?, \n            maybe_frame_width = ?, \n            maybe_frame_height = ?,\n            maybe_prompt_token = ?,\n            checksum_sha2 = ?,\n\n            maybe_title = ?,\n            maybe_text_transcript = ?,\n\n            maybe_scene_source_media_file_token = ?,\n\n            public_bucket_directory_hash = ?, \n            maybe_public_bucket_prefix = ?, \n            maybe_public_bucket_extension = ?, \n\n            maybe_creator_user_token = ?, \n            maybe_creator_anonymous_visitor_token = ?, \n\n            creator_ip_address = ?, \n            creator_set_visibility = ?, \n\n            maybe_creator_file_synthetic_id = ?, \n            maybe_creator_category_synthetic_id = ?,\n\n            extra_file_modification_info = ?,\n\n            maybe_mod_user_token = ?, \n            is_generated_on_prem = ?, \n            generated_by_worker = ?, \n            generated_by_cluster = ?\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 38
+    },
+    "nullable": []
+  },
+  "hash": "c58f414b5e4d0f0420d1894e2f40311e5613919f49f3ac91f81f5fc2bef1bea8"
+}

.sqlx/query-dc0367e9e43c3d7dda3623e77337ab4529eac2192475be7d08589886703ba1cb.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "\n    UPDATE tasks\n    SET task_status = ?\n    WHERE id = ?\n  ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 2
-    },
-    "nullable": []
-  },
-  "hash": "dc0367e9e43c3d7dda3623e77337ab4529eac2192475be7d08589886703ba1cb"
-}

.sqlx/query-f68a8917e639418a0bb952bef606c96299001e143c745ee30df99b72d1235d35.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "MySQL",
-  "query": "\nUPDATE generic_inference_jobs\nSET\n  status = ?,\n  on_success_result_entity_type = ?,\n  on_success_result_entity_token = ?,\n  failure_reason = NULL,\n  internal_debugging_failure_reason = NULL,\n  success_execution_millis = NULL,\n  success_inference_execution_millis = NULL,\n  retry_at = NULL,\n  successfully_completed_at = NOW()\nWHERE token = ?\n        ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 4
-    },
-    "nullable": []
-  },
-  "hash": "f68a8917e639418a0bb952bef606c96299001e143c745ee30df99b72d1235d35"
-}

.sqlx/query-fbe482c8987231c3b5d75f2e792d0fdd033de8a7eb821428fa034752c73edfb7.json

@@ -1,124 +0,0 @@
-{
-  "db_name": "MySQL",
-  "query": "\nSELECT\n    jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,\n\n    jobs.status as `status: enums::common::job_status_plus::JobStatusPlus`,\n    \n    jobs.maybe_external_third_party as `external_third_party: enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty`,\n    jobs.maybe_external_third_party_id as `external_third_party_id`,\n\n    jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n    jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,\n    jobs.creator_ip_address,\n    \n    jobs.maybe_prompt_token as `maybe_prompt_token: tokens::tokens::prompts::PromptToken`,\n\n    jobs.created_at,\n    jobs.updated_at\n\nFROM generic_inference_jobs as jobs\n\nWHERE jobs.maybe_external_third_party = ?\nAND jobs.maybe_external_third_party_id = ?\n        ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken",
-        "type_info": {
-          "type": "VarString",
-          "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
-          "char_set": 224,
-          "max_size": 128
-        }
-      },
-      {
-        "ordinal": 1,
-        "name": "status: enums::common::job_status_plus::JobStatusPlus",
-        "type_info": {
-          "type": "String",
-          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | ENUM",
-          "char_set": 224,
-          "max_size": 76
-        }
-      },
-      {
-        "ordinal": 2,
-        "name": "external_third_party: enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty",
-        "type_info": {
-          "type": "VarString",
-          "flags": "MULTIPLE_KEY | BINARY",
-          "char_set": 224,
-          "max_size": 64
-        }
-      },
-      {
-        "ordinal": 3,
-        "name": "external_third_party_id",
-        "type_info": {
-          "type": "VarString",
-          "flags": "MULTIPLE_KEY | BINARY",
-          "char_set": 224,
-          "max_size": 256
-        }
-      },
-      {
-        "ordinal": 4,
-        "name": "maybe_creator_user_token: tokens::tokens::users::UserToken",
-        "type_info": {
-          "type": "VarString",
-          "flags": "MULTIPLE_KEY | BINARY",
-          "char_set": 224,
-          "max_size": 128
-        }
-      },
-      {
-        "ordinal": 5,
-        "name": "maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken",
-        "type_info": {
-          "type": "VarString",
-          "flags": "MULTIPLE_KEY | BINARY",
-          "char_set": 224,
-          "max_size": 128
-        }
-      },
-      {
-        "ordinal": 6,
-        "name": "creator_ip_address",
-        "type_info": {
-          "type": "VarString",
-          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
-          "char_set": 224,
-          "max_size": 160
-        }
-      },
-      {
-        "ordinal": 7,
-        "name": "maybe_prompt_token: tokens::tokens::prompts::PromptToken",
-        "type_info": {
-          "type": "VarString",
-          "flags": "BINARY",
-          "char_set": 224,
-          "max_size": 128
-        }
-      },
-      {
-        "ordinal": 8,
-        "name": "created_at",
-        "type_info": {
-          "type": "Timestamp",
-          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP",
-          "char_set": 63,
-          "max_size": 19
-        }
-      },
-      {
-        "ordinal": 9,
-        "name": "updated_at",
-        "type_info": {
-          "type": "Timestamp",
-          "flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
-          "char_set": 63,
-          "max_size": 19
-        }
-      }
-    ],
-    "parameters": {
-      "Right": 2
-    },
-    "nullable": [
-      false,
-      false,
-      true,
-      true,
-      true,
-      true,
-      false,
-      true,
-      false,
-      false
-    ]
-  },
-  "hash": "fbe482c8987231c3b5d75f2e792d0fdd033de8a7eb821428fa034752c73edfb7"
-}

.vscode/settings.json

@@ -1,15 +1,6 @@
 {
-  "files.exclude": {
-    "**/*~": true,
-    ".sqlx/**": true,
-    "_elasticsearch/**": true,
-    "_sql/**": true,
-    "frontend_old/**": true,
-    "includes/**": true,
-    "target/**": true,
-    "test_data/**": true,
-    "tmp/**": true,
-    "virtualenv/**": true,
-    "windows_dlls/**": true,
-  }
+    "files.exclude": {
+        "**/*~": true,
+        "frontend_old/**": true
+    }
 }

Cargo.toml

@@ -4,17 +4,12 @@ resolver = "2"
 # This block defines the workspace members.
 # When we add a library or binary crate, it needs an entry here.
 members = [
-  # API clients for external services
-  # These might be custom implementations or wrappers over other client libraries.
-  "crates/api_clients/cloud_storage",
-  "crates/api_clients/fal_client",
-  "crates/api_clients/google_drive_common",
-  "crates/api_clients/google_sign_in",
-  "crates/api_clients/openai_sora_client",
-  "crates/api_clients/openart_client",
-  "crates/api_clients/runwayml_client",
-  "crates/api_clients/storyteller_client",
-  "crates/api_clients/thumbnail_generator",
+  # Types: common strongly-typed tokens, ids, payloads, etc.
+  "crates/_types/composite_identifiers",
+  "crates/_types/enums",
+  "crates/_types/enums_public",
+  "crates/_types/reusable_types", # TODO(bt, 2022-12-21): Deprecate
+  "crates/_types/tokens",
 
   # CLI tools for operations, etc.
   "crates/cli/devenv/dev-database-seed",
@@ -34,28 +29,31 @@ members = [
   # Shared libraries
   "crates/lib/actix_cors_configs",
   "crates/lib/actix_helpers",
+  "crates/lib/api_clients/cloud_storage",
+  "crates/lib/api_clients/google_drive_common",
+  "crates/lib/api_clients/google_sign_in",
+  "crates/lib/api_clients/openai_sora_client",
+  "crates/lib/api_clients/storyteller_client",
+  "crates/lib/api_clients/thumbnail_generator",
   "crates/lib/bootstrap",
   "crates/lib/bucket_paths",
-  "crates/lib/build_metadata",
   "crates/lib/caching/memory_caching",
   "crates/lib/caching/memory_store",
   "crates/lib/caching/redis_caching",
-  "crates/lib/cloudflare_errors",
   "crates/lib/concurrency",
   "crates/lib/cookies",
   "crates/lib/data/collections",
   "crates/lib/data/crockford",
   "crates/lib/data/datetimes",
   "crates/lib/data/primitives",
-  "crates/lib/data/web_base64",
   "crates/lib/deprecated/config", # TODO(bt, 2023-02-13): Deprecate or repurpose
   "crates/lib/deprecated/http_server_common", # TODO(bt, 2023-03-20): Move useful pieces to actix_helpers
   "crates/lib/dimensions",
   "crates/lib/easyenv", # NB: Vendored version of https://github.com/echelon/easyenv.rs that we'll improve.
+  "crates/lib/elasticsearch_schema",
   "crates/lib/email_sender",
   "crates/lib/errors",
   "crates/lib/files/archives",
-  "crates/lib/files/file_to_base64_url",
   "crates/lib/files/filesys",
   "crates/lib/files/hashing",
   "crates/lib/files/images",
@@ -65,42 +63,28 @@ members = [
   "crates/lib/idempotency",
   "crates/lib/image_algorithms",
   "crates/lib/jobs_common",
-  "crates/lib/jwt_light",
   "crates/lib/logging",
+  "crates/lib/machine_learning/ml_models",
   "crates/lib/machine_learning/ml_weights_registry",
   "crates/lib/machine_learning/prompt_engineering",
   "crates/lib/markdown",
   "crates/lib/migration",
+  "crates/lib/mysql_queries",
   "crates/lib/password",
+  "crates/lib/redis_common",
+  "crates/lib/redis_schema",
   "crates/lib/server_environment",
   "crates/lib/shared_service_components",
   "crates/lib/sqlx_mysql_helpers",
   "crates/lib/storyteller_root",
   "crates/lib/subprocess_common",
-  "crates/lib/test_data",
   "crates/lib/testing",
   "crates/lib/tts_common",
   "crates/lib/url_config",
   "crates/lib/user_input_common",
   "crates/lib/web_scrapers",
   #"crates/lib/api_clients/fakeyou_client",
-  #"crates/lib/machine_learning/ml_models",
-
-  # Schema (database): queries, database layouts, Redis keys, etc.
-  "crates/schema/database/elasticsearch_schema",
-  "crates/schema/database/mysql_queries",
-  "crates/schema/database/redis_common",
-  "crates/schema/database/redis_schema",
-  "crates/schema/database/sqlite_tasks",
-  #"crates/schema/database/sqlite_queries",
-
-  # Schema (public): common strongly-typed tokens, ids, payloads, etc.
-  "crates/schema/public/api/artcraft_api_defs",
-  "crates/schema/public/composite_identifiers",
-  "crates/schema/public/enums",
-  "crates/schema/public/enums_public",
-  "crates/schema/public/reusable_types", # TODO(bt, 2022-12-21): Deprecate
-  "crates/schema/public/tokens",
+  #"crates/lib/sqlite_queries",
 
   # Services: API servers, jobs, workers, etc.
   "crates/service/job/analytics_job",
@@ -121,23 +105,19 @@ members = [
   "crates/service/plugins/user_traits_component",
 
   # Vendored code
-  "crates/vendor/fal-rs",
-  "crates/vendor/fal-rs/fal-derive",
-  "crates/vendor/fal-rs/generate",
+  "crates/vendor/candle/candle-core",
+  "crates/vendor/candle/candle-datasets",
+  "crates/vendor/candle/candle-examples",
+  "crates/vendor/candle/candle-flash-attn",
+  "crates/vendor/candle/candle-kernels",
+  "crates/vendor/candle/candle-metal-kernels",
+  "crates/vendor/candle/candle-nn",
+  "crates/vendor/candle/candle-onnx",
+  "crates/vendor/candle/candle-pyo3",
+  "crates/vendor/candle/candle-transformers",
+  "crates/vendor/candle/tensor-tools",
   "crates/vendor/limitation",
   "crates/vendor/tauri-plugin-http",
-  # NB(bt,2025-05-25): Too many failures on Mac. Disable for now.
-  #"crates/vendor/candle/candle-core",
-  #"crates/vendor/candle/candle-datasets",
-  #"crates/vendor/candle/candle-examples",
-  #"crates/vendor/candle/candle-flash-attn",
-  #"crates/vendor/candle/candle-kernels",
-  #"crates/vendor/candle/candle-metal-kernels",
-  #"crates/vendor/candle/candle-nn",
-  #"crates/vendor/candle/candle-onnx",
-  #"crates/vendor/candle/candle-pyo3",
-  #"crates/vendor/candle/candle-transformers",
-  #"crates/vendor/candle/tensor-tools",
   # NB(bt,2023-12-12): Newrelic keeps breaking with upgrades. Kill for now.
   #"crates/vendor/newrelic-telemetry-sdk-rust",
 
@@ -177,66 +157,41 @@ workspace-build-acceleration = { path = "crates/workspace-build-acceleration" }
 # External libs
 # Centrally pinned versions so that we can uniformly upgrade/downgrade versions across the workspace
 # If a crate needs to use an older/newer version, the dependency should be managed in that crate's own Cargo.toml
-actix = "0.13.5"
-actix-cors = "0.6.4" # TODO(bt): Carefully upgrade this. CORS broke last time.
-actix-http = "3.7.0"
-actix-multipart = "0.6.2"
-actix-rt = "2.10.0"
-actix-service = "2.0.3"
-actix-utils = "3.0.1"
-actix-web = "4.7.0" # TODO(bt): We're behind and need to bump versions.
-actix-web-actors = "4.3.1+deprecated"
-actix-web-lab = "0.20.2"
-anyhow = "1.0.98"
+anyhow = "1.0.97"
 base64 = "0.22.1"
-bytes = "1.10.1"
-chrono = { version = "0.4.41", features = ["serde"] }
+chrono = { version = "0.4.40", features = ["serde"] }
 cookie = "0.18.1"
-futures = { version = "0.3.31" }
 hex-literal = "1.0.0"
 indexmap = "2.9.0"
 log = "0.4.27"
 once_cell = "1.21.3"
-rand = "0.9.1"
+rand = "0.9.0"
 regex = "1.11.1"
 reqwest = { version = "0.12.15", features = ["json", "cookies"] }
 serde = {  version = "1.0.219", features = ["derive"] }
 serde_derive = "1.0.219"
 serde_json = "1.0.140"
-serde_yaml = "0.9.34+deprecated"
+serde_yaml = "0.9.1"
 sha3 = "0.10.8"
 tauri = { version = "2.4.1", features = ["unstable", "devtools"] } # NB: "devtools" prevents App Store distribution
-tempdir = "0.3.7"
-tempfile = "3.20.0"
 thiserror = "2.0.12"
-tokio = { version = "1.45.1", features = ["macros"] }
-toml = "0.8.22"
+toml = "0.8.20"
 url = { version = "2.5.4", features = ["serde"] }
 
 # Internal libs
-artcraft_api_defs = { path = "crates/schema/public/api/artcraft_api_defs" }
-build_metadata = { path = "crates/lib/build_metadata" }
-cloud_storage = { path = "crates/api_clients/cloud_storage" }
-cloudflare_errors = { path = "crates/lib/cloudflare_errors" }
 crockford = { path = "crates/lib/data/crockford" }
-enums = { path = "crates/schema/public/enums" }
-enums_public = { path = "crates/schema/public/enums_public" }
+enums = { path = "crates/_types/enums" }
+enums_public = { path = "crates/_types/enums_public" }
 errors = { path = "crates/lib/errors" }
-fal_client = { path = "crates/api_clients/fal_client" }
-file_to_base64_url = { path = "crates/lib/files/file_to_base64_url" }
 filesys = { path = "crates/lib/files/filesys" }
 idempotency = { path = "crates/lib/idempotency" }
 images = { path = "crates/lib/files/images" }
-jwt_light = { path = "crates/lib/jwt_light" }
 memory_caching = { path = "crates/lib/caching/memory_caching" }
 memory_store = { path = "crates/lib/caching/memory_store" }
 mimetypes = { path = "crates/lib/files/mimetypes" }
-openai_sora_client = { path = "crates/api_clients/openai_sora_client" }
+openai_sora_client = { path = "crates/lib/api_clients/openai_sora_client" }
 primitives = { path = "crates/lib/data/primitives" }
 shared_service_components = { path = "crates/lib/shared_service_components" }
-sqlite_tasks = { path = "crates/schema/database/sqlite_tasks" }
-storyteller_client = { path = "crates/api_clients/storyteller_client" }
+storyteller_client = { path = "crates/lib/api_clients/storyteller_client" }
 subprocess_common = { path = "crates/lib/subprocess_common" }
-test_data = { path = "crates/lib/test_data" }
-tokens = { path = "crates/schema/public/tokens" }
-web_base64 = { path = "crates/lib/data/web_base64" }
+tokens = { path = "crates/_types/tokens" }

README.md

@@ -1,7 +1,7 @@
 storyteller-rust
 ================
 
-Our Artcraft monorepo, which serves our desktop application, backend infrastructure, and more.
+Our Rust monorepo, which serves both our API surface area and our downloadable client applications.
 
 ```
                  ."-,.__
@@ -45,6 +45,5 @@ Our Artcraft monorepo, which serves our desktop application, backend infrastruct
                  / ,"'"\,'               `/  `-.|" 
 ```
 
-- [developer documentation](./_docs)
-- [server API reference](https://storyteller-docs.netlify.app/)
-- [tools, scripts, misc](./script)
+- [documentation](./_develop/docs)
+- [tools, misc](./_develop)

_develop/docs/local_development/general_setup.md

@@ -0,0 +1,395 @@
+FakeYou Development Setup (bare metal)
+======================================
+
+This document describes how to run the FakeYou infrastructure on your development 
+machine's bare metal (no containers).
+
+Setup instructions and development were tested and known to work on the following
+platforms:
+
+- Mac OS (Apple M2 silicon)
+- Ubuntu 22.04
+- Windows Subsystem for Linux (Ubuntu 22.04 distro) aka "WSL"
+
+The applications (binary targets) you might be interested in running include, but 
+are not limited to:
+
+- `storyteller-web`, the HTTP API server
+- `download-job`, which downloads models from the web and stores them in GCS
+- `inference-job`, which downloads models from GCS and runs inference with them
+
+Setup Instructions
+------------------
+
+### (1) Install Rust
+
+Install Rust [using this guide](https://www.rust-lang.org/learn/get-started). If it asks, 
+you'll want "stable" Rust, not "nightly" Rust. If it doesn't ask, it defaults to "stable".
+
+For Windows Subsystem for Linux, you'll want the WSL version, not the `.exe` file!
+
+### (2) Install the necessary libraries
+
+#### Mac
+
+On an up-to-date M1/M2 Mac, there should be no libraries you need to install! Any exceptions 
+to this assumption should be documented here.
+
+#### Ubuntu 22.04 and WSL:
+
+If you're on Ubuntu or WSL, you'll need to install some libraries to build the Rust code: 
+
+```bash
+# If you haven't run apt before, you'll need to fetch the package list:
+sudo apt update
+
+# Install the dependencies
+sudo apt install jq \
+  libmysqlclient-dev \
+  libsqlite3-dev \
+  pkgconf
+```
+
+(Note: jq is for combining mysql codegen outputs; pkgconf is if using openssl instead of rustls.)
+
+### (3) Install MySQL server
+
+#### Mac
+
+```bash
+# Install MySQL
+brew install mysql
+
+# Try to connect
+mysql -uroot
+```
+
+In the event of failure to connect, try the following steps:
+
+```bash
+# If connection fails, start service
+brew services start mysql
+
+# Try to connect again 
+# If this fails, reboot the machine, eg. using `sudo reboot now`
+mysql -uroot
+```
+
+#### Ubuntu 22.04 and WSL
+
+```bash
+# Install MySQL
+sudo apt install mysql-server
+
+# Try to connect 
+# The default password is typically "root".
+sudo mysql -u root -p
+````
+
+In the event of failure to connect, try the following steps:
+
+```bash
+# If it failed, you may need to start MySQL (typically only on WSL, not on native Ubuntu)
+sudo service mysql start
+
+# You may also have to change some socket file permissions (again, typically only on WSL)
+sudo chmod g+rx /var/run/mysqld
+sudo usermod -aG mysql $USER
+newgrp mysql
+
+# Try to connect again
+# The default password is typically "root".
+# If this fails, try rebooting. If it still fails, read some of the suggestions at the bottom 
+# of this README.
+sudo mysql -u root -p
+```
+
+### (4) Install a `storyteller` user and table in MySQL
+
+First, connect to MySQL:
+
+```bash
+# If the following command asks for a password, the password is typically "root"
+sudo mysql -u root -p
+```
+
+Once in MySQL, run the following to set up the database and user accounts:
+
+```mysql
+use mysql;
+CREATE DATABASE storyteller;
+CREATE USER 'storyteller'@'localhost' IDENTIFIED BY 'password';
+GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';
+```
+
+Then verify access with `./dev_mysql_connect.sh`
+
+### (5) Install Diesel CLI (for MySQL migrations):
+
+In order to run MySQL database migrations, you'll need to install the migration tool we use:
+
+```bash
+cargo install diesel_cli \
+  --no-default-features \
+  --features mysql,sqlite
+```
+
+In older Macs, there [have been some issues with Diesel CLI](https://github.com/diesel-rs/diesel/issues/2605)
+that require a few extra dependencies to be installed:
+
+```bash
+# If you're on Mac and the above command didn't work, run the following and then retry:
+brew install libpq
+```
+
+### (6) Run the pending database migrations:
+
+You're ready to run the migrations:
+
+```bash
+diesel migration run
+```
+
+You might get a scary message about `"Encountered unknown type for Mysql: enum"` -- you can safely ignore this 
+error if you see it in isolation. It doesn't impact the migrations whatsoever. (See notes at the bottom of 
+the README for details.)
+
+### (7) (optional) Install the SQLx CLI (if doing database development)
+
+(You can probably skip this step unless you're a backend engineer.) 
+
+If (and only if) you're doing Rust service development that changes the MySQL queries, then you should install 
+SQLx utilities. This tool is used to cache the column types and query plans for CI, as SQLx statically verifies 
+raw MySQL queries against the schema and Rust types.
+
+We'll be using diesel to manage the migrations, but sqlx within the app to actually perform queries.
+Diesel is an immature ORM, which isn't a good tech bet, so we use sqlx as at-compile-time
+typesafe SQL. 
+
+(TODO: Reevaluate this assessment for 2023-2024 and beyond. Maybe we do want Diesel now.)
+
+```bash
+# NB(bt,2023-10-04): "--no-default-features" not working
+# cargo install sqlx-cli --no-default-features --features rustls,mysql,sqlite
+cargo install sqlx-cli --features rustls,mysql,sqlite
+```
+
+### (8) Install hosts file:
+
+(This is the same for Linux, Mac, and WSL.)
+
+If you're developing against the frontend, it'll target development domains (eg. `dev.fakeyou.com`) instead 
+of `127.0.0.1` or `localhost`. You can make your machine route domains to localhost by editing your hosts 
+file (located at `/etc/hosts`) to include the following configuration lines:
+
+```
+# 1) Edit hosts file with your editor of choice
+#
+#  - sudo vi /etc/hosts
+#  - sudo nano /etc/hosts
+#
+# 2) Then paste:
+#
+
+127.0.0.1    dev.fakeyou.com
+127.0.0.1    api.dev.fakeyou.com
+127.0.0.1    devproxy.fakeyou.com
+
+127.0.0.1    dev.storyteller.ai
+127.0.0.1    api.dev.storyteller.ai
+127.0.0.1    devproxy.storyteller.ai
+```
+
+### (9) Install Redis
+
+#### Mac:
+
+```bash
+# Install Redis
+brew install redis
+
+# Start the service:
+brew services start redis
+```
+
+#### Ubuntu 22.04 and WSL
+
+```
+# Install Redis
+sudo apt install redis
+
+# Start the service (typically only needed on WSL):
+sudo service redis-server start
+```
+
+### (10) Install Elasticsearch (optional)
+
+(You can most likely skip this step.)
+
+Mac: (TODO - haven't installed yet)
+
+WSL: (TODO - haven't installed yet)
+
+Ubuntu 22.04:
+
+Install:
+
+```bash
+curl -fsSL https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo gpg --dearmor -o /usr/share/keyrings/elastic.gpg
+
+echo "deb [signed-by=/usr/share/keyrings/elastic.gpg] https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list
+
+sudo apt update
+sudo apt install elasticsearch
+```
+
+Configure:
+
+```text
+# Edit /etc/elasticsearch/elasticsearch.yml
+# Set the following:
+network.host: localhost
+```
+
+Run:
+
+```bash
+sudo systemctl start elasticsearch
+sudo systemctl enable elasticsearch
+```
+
+Verify:
+
+```bash
+curl -X GET 'http://localhost:9200'
+```
+
+Install Kibana (prototyping UI):
+
+```bash
+# Install
+sudo apt-get install kibana
+
+# Run
+sudo /bin/systemctl daemon-reload
+sudo /bin/systemctl enable kibana.service
+sudo systemctl start kibana.service
+
+# Access the Kibana Status Page:
+# http://localhost:5601/status
+
+# Access the query console UI:
+# Click Sidebar > Management > Dev Tools > Console
+# http://localhost:5601/
+```
+
+Extra reading on setting up Elasticsearch: 
+
+- [Guide 1](https://www.digitalocean.com/community/tutorials/how-to-install-and-configure-elasticsearch-on-ubuntu-22-04)
+- [Guide 2](https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html)
+- [Kibana](https://www.elastic.co/guide/en/kibana/current/deb.html)
+
+### (11) Run one or more applications:
+
+To start the HTTP API server,
+
+```bash
+cargo run --bin storyteller-web
+```
+
+Note that this compiles and runs the "development" binary. It's faster and easier to debug than the
+optimized "production" build. To build a fully optimized production release,
+run `cargo build --release --bin storyteller-web` . Note that this will take much longer.
+
+You may notice that this might not work right off the bat. You may need to specify some environment variables 
+and supply some development secrets.
+
+Secrets: 
+
+- Global secrets (ask Brandon for these): 
+  - Put `.env-secrets` in the root `storyteller-rust` directory.
+  
+- Per app secrets (ask Brandon for these):
+  - Put `storteller-web.development-secrets.env` in `crates/service/web/storyteller_web/config`
+  - Put `inference-job.development-secrets.env` in `crates/service/job/inference_job/config`
+  - Put `donwload-job.development-secrets.env` in `crates/service/job/download_job/config`
+
+**DO NOT CHECK THESE SECRET FILES INTO THE REPO!**
+
+Environment config: 
+
+You may need to set local development paths for your environment. You can set these in `.env-secrets` too,
+so that they won't be updated for other engineers.
+
+```
+# this should be the directory containing all of the "storyteller" projects:
+STORYTELLER_ROOT = "/home/tensor/code/storyteller" # change this
+
+# The following are entirely optional, but if you want to fine tune where each project lives, you may do so:
+STORYTELLER_FRONTEND = "/somewhere/else/storyteller-frontend"
+STORYTELLER_ML = "storyteller-ml"
+STORYTELLER_RUST = "/Users/bt/Development/storyteller-rust"
+```
+
+To download some ML models, run:
+
+```bash
+cargo run --bin download-job
+cargo run --bin tts-download-job
+```
+
+If you want to run both the HTTP API and the jobs, you'll need to run both processes in different
+terminals or tmux sessions.
+
+Note also that the configurations pointing to the ML monorepo must be set up for each application.
+Additionally, some development secrets may be needed (ask the team to share).
+
+To execute ML inference, run:
+
+```bash
+cargo run --bin inference-job
+cargo run --bin tts-inference-job
+```
+
+Again, note that the configurations pointing to the ML monorepo must be set up for each application.
+Additionally, some development secrets may be needed (ask the team to share).
+
+Seeding the Database with Some Values
+-------------------------------------
+For voice designer you might want some pre-seeded audio that will hit the google cloud bucket, it will require env-secrets ask Brandon.
+cargo run --bin dev-database-seed -- --bucket
+
+Solutions to Common Setup Problems 
+-----------------------------------
+
+### [Fix] Sqlx Error when Performing Schema Migrations
+
+You might get this error message during migration,
+
+```
+Encountered unknown type for Mysql: enum
+thread 'main' panicked at 'internal error: entered unreachable code: Mysql only supports a closed set of types.
+                         If you ever see this error message please open an issue at https://github.com/diesel-rs/diesel 
+                         containing a dump of your schema definition.', diesel_cli/src/print_schema.rs:310:17
+note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
+```
+
+I haven't found the cause (it doesn't happen on newer installs), but the migrations appear to work regardless 
+of this error message. You can essentially ignore it.
+
+### [Fix] Can't connect to local MySQL after install on Ubuntu
+
+- If MySql in local dev can't be connected to, [reset the accounts](https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux).
+
+### API Server
+```
+cargo run --bin docs
+```
+This setups an api server with swagger docs to show you responses and requests with each associated endpoint
+It is run at http://127.0.0.1:8989/
+To obtain the json for the postman collection go to this url http://127.0.0.1:8989/api-docs/openapi.json
+We use this: `https://github.com/juhaku/utoipa`
+The documentation example is in: 
+`crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs`
+Write into the api doc to: 
+`crates/service/web/storyteller_web/src/api_doc.rs`

_docs/code_conventions.md

@@ -1,26 +0,0 @@
-Code Conventions
-================
-
-We use three languages at Artcraft: Rust, TypeScript, and Python.
-
-## Rust style guide
-
-We deviate from the official Rust style guide in a few places:
-
-- We use two spaces for indentation instead of four.
-- We typically keep one function or type definition per file, though this isn't a steadfast rule.
-- We use a Cargo workspace to share code within our monorepo.
-
-### Rust Preferred Libraries
-
-We use the following Rust libraries, though we are open to migrating to alternatives:
-
-- `actix-web` for our HTTP web server
-- `reqwest` for HTTP client requests
-- `sqlx` for SQL database operations
-- `tokio` for asynchronous programming
-
-## TypeScript style guide
-
-- We use two spaces for indentation instead of four.
-- We use `nx` for monorepo management

_docs/local_development_setup_guide.md

@@ -1,395 +0,0 @@
-Local Development Setup Guide (for bare metal)
-==============================================
-
-This document describes how to run the FakeYou infrastructure on your development 
-machine's bare metal (no containers).
-
-Setup instructions and development were tested and known to work on the following
-platforms:
-
-- Mac OS (Apple M2 silicon)
-- Ubuntu 22.04
-- Windows Subsystem for Linux (Ubuntu 22.04 distro) aka "WSL"
-
-The applications (binary targets) you might be interested in running include, but 
-are not limited to:
-
-- `storyteller-web`, the HTTP API server
-- `download-job`, which downloads models from the web and stores them in GCS
-- `inference-job`, which downloads models from GCS and runs inference with them
-
-Setup Instructions
-------------------
-
-### (1) Install Rust
-
-Install Rust [using this guide](https://www.rust-lang.org/learn/get-started). If it asks, 
-you'll want "stable" Rust, not "nightly" Rust. If it doesn't ask, it defaults to "stable".
-
-For Windows Subsystem for Linux, you'll want the WSL version, not the `.exe` file!
-
-### (2) Install the necessary libraries
-
-#### Mac
-
-On an up-to-date M1/M2 Mac, there should be no libraries you need to install! Any exceptions 
-to this assumption should be documented here.
-
-#### Ubuntu 22.04 and WSL:
-
-If you're on Ubuntu or WSL, you'll need to install some libraries to build the Rust code: 
-
-```bash
-# If you haven't run apt before, you'll need to fetch the package list:
-sudo apt update
-
-# Install the dependencies
-sudo apt install jq \
-  libmysqlclient-dev \
-  libsqlite3-dev \
-  pkgconf
-```
-
-(Note: jq is for combining mysql codegen outputs; pkgconf is if using openssl instead of rustls.)
-
-### (3) Install MySQL server
-
-#### Mac
-
-```bash
-# Install MySQL
-brew install mysql
-
-# Try to connect
-mysql -uroot
-```
-
-In the event of failure to connect, try the following steps:
-
-```bash
-# If connection fails, start service
-brew services start mysql
-
-# Try to connect again 
-# If this fails, reboot the machine, eg. using `sudo reboot now`
-mysql -uroot
-```
-
-#### Ubuntu 22.04 and WSL
-
-```bash
-# Install MySQL
-sudo apt install mysql-server
-
-# Try to connect 
-# The default password is typically "root".
-sudo mysql -u root -p
-````
-
-In the event of failure to connect, try the following steps:
-
-```bash
-# If it failed, you may need to start MySQL (typically only on WSL, not on native Ubuntu)
-sudo service mysql start
-
-# You may also have to change some socket file permissions (again, typically only on WSL)
-sudo chmod g+rx /var/run/mysqld
-sudo usermod -aG mysql $USER
-newgrp mysql
-
-# Try to connect again
-# The default password is typically "root".
-# If this fails, try rebooting. If it still fails, read some of the suggestions at the bottom 
-# of this README.
-sudo mysql -u root -p
-```
-
-### (4) Install a `storyteller` user and table in MySQL
-
-First, connect to MySQL:
-
-```bash
-# If the following command asks for a password, the password is typically "root"
-sudo mysql -u root -p
-```
-
-Once in MySQL, run the following to set up the database and user accounts:
-
-```mysql
-use mysql;
-CREATE DATABASE storyteller;
-CREATE USER 'storyteller'@'localhost' IDENTIFIED BY 'password';
-GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';
-```
-
-Then verify access with `./dev_mysql_connect.sh`
-
-### (5) Install Diesel CLI (for MySQL migrations):
-
-In order to run MySQL database migrations, you'll need to install the migration tool we use:
-
-```bash
-cargo install diesel_cli \
-  --no-default-features \
-  --features mysql,sqlite
-```
-
-In older Macs, there [have been some issues with Diesel CLI](https://github.com/diesel-rs/diesel/issues/2605)
-that require a few extra dependencies to be installed:
-
-```bash
-# If you're on Mac and the above command didn't work, run the following and then retry:
-brew install libpq
-```
-
-### (6) Run the pending database migrations:
-
-You're ready to run the migrations:
-
-```bash
-diesel migration run
-```
-
-You might get a scary message about `"Encountered unknown type for Mysql: enum"` -- you can safely ignore this 
-error if you see it in isolation. It doesn't impact the migrations whatsoever. (See notes at the bottom of 
-the README for details.)
-
-### (7) (optional) Install the SQLx CLI (if doing database development)
-
-(You can probably skip this step unless you're a backend engineer.) 
-
-If (and only if) you're doing Rust service development that changes the MySQL queries, then you should install 
-SQLx utilities. This tool is used to cache the column types and query plans for CI, as SQLx statically verifies 
-raw MySQL queries against the schema and Rust types.
-
-We'll be using diesel to manage the migrations, but sqlx within the app to actually perform queries.
-Diesel is an immature ORM, which isn't a good tech bet, so we use sqlx as at-compile-time
-typesafe SQL. 
-
-(TODO: Reevaluate this assessment for 2023-2024 and beyond. Maybe we do want Diesel now.)
-
-```bash
-# NB(bt,2023-10-04): "--no-default-features" not working
-# cargo install sqlx-cli --no-default-features --features rustls,mysql,sqlite
-cargo install sqlx-cli --features rustls,mysql,sqlite
-```
-
-### (8) Install hosts file:
-
-(This is the same for Linux, Mac, and WSL.)
-
-If you're developing against the frontend, it'll target development domains (eg. `dev.fakeyou.com`) instead 
-of `127.0.0.1` or `localhost`. You can make your machine route domains to localhost by editing your hosts 
-file (located at `/etc/hosts`) to include the following configuration lines:
-
-```
-# 1) Edit hosts file with your editor of choice
-#
-#  - sudo vi /etc/hosts
-#  - sudo nano /etc/hosts
-#
-# 2) Then paste:
-#
-
-127.0.0.1    dev.fakeyou.com
-127.0.0.1    api.dev.fakeyou.com
-127.0.0.1    devproxy.fakeyou.com
-
-127.0.0.1    dev.storyteller.ai
-127.0.0.1    api.dev.storyteller.ai
-127.0.0.1    devproxy.storyteller.ai
-```
-
-### (9) Install Redis
-
-#### Mac:
-
-```bash
-# Install Redis
-brew install redis
-
-# Start the service:
-brew services start redis
-```
-
-#### Ubuntu 22.04 and WSL
-
-```
-# Install Redis
-sudo apt install redis
-
-# Start the service (typically only needed on WSL):
-sudo service redis-server start
-```
-
-### (10) Install Elasticsearch (optional)
-
-(You can most likely skip this step.)
-
-Mac: (TODO - haven't installed yet)
-
-WSL: (TODO - haven't installed yet)
-
-Ubuntu 22.04:
-
-Install:
-
-```bash
-curl -fsSL https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo gpg --dearmor -o /usr/share/keyrings/elastic.gpg
-
-echo "deb [signed-by=/usr/share/keyrings/elastic.gpg] https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list
-
-sudo apt update
-sudo apt install elasticsearch
-```
-
-Configure:
-
-```text
-# Edit /etc/elasticsearch/elasticsearch.yml
-# Set the following:
-network.host: localhost
-```
-
-Run:
-
-```bash
-sudo systemctl start elasticsearch
-sudo systemctl enable elasticsearch
-```
-
-Verify:
-
-```bash
-curl -X GET 'http://localhost:9200'
-```
-
-Install Kibana (prototyping UI):
-
-```bash
-# Install
-sudo apt-get install kibana
-
-# Run
-sudo /bin/systemctl daemon-reload
-sudo /bin/systemctl enable kibana.service
-sudo systemctl start kibana.service
-
-# Access the Kibana Status Page:
-# http://localhost:5601/status
-
-# Access the query console UI:
-# Click Sidebar > Management > Dev Tools > Console
-# http://localhost:5601/
-```
-
-Extra reading on setting up Elasticsearch: 
-
-- [Guide 1](https://www.digitalocean.com/community/tutorials/how-to-install-and-configure-elasticsearch-on-ubuntu-22-04)
-- [Guide 2](https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html)
-- [Kibana](https://www.elastic.co/guide/en/kibana/current/deb.html)
-
-### (11) Run one or more applications:
-
-To start the HTTP API server,
-
-```bash
-cargo run --bin storyteller-web
-```
-
-Note that this compiles and runs the "development" binary. It's faster and easier to debug than the
-optimized "production" build. To build a fully optimized production release,
-run `cargo build --release --bin storyteller-web` . Note that this will take much longer.
-
-You may notice that this might not work right off the bat. You may need to specify some environment variables 
-and supply some development secrets.
-
-Secrets: 
-
-- Global secrets (ask Brandon for these): 
-  - Put `.env-secrets` in the root `storyteller-rust` directory.
-  
-- Per app secrets (ask Brandon for these):
-  - Put `storteller-web.development-secrets.env` in `crates/service/web/storyteller_web/config`
-  - Put `inference-job.development-secrets.env` in `crates/service/job/inference_job/config`
-  - Put `donwload-job.development-secrets.env` in `crates/service/job/download_job/config`
-
-**DO NOT CHECK THESE SECRET FILES INTO THE REPO!**
-
-Environment config: 
-
-You may need to set local development paths for your environment. You can set these in `.env-secrets` too,
-so that they won't be updated for other engineers.
-
-```
-# this should be the directory containing all of the "storyteller" projects:
-STORYTELLER_ROOT = "/home/tensor/code/storyteller" # change this
-
-# The following are entirely optional, but if you want to fine tune where each project lives, you may do so:
-STORYTELLER_FRONTEND = "/somewhere/else/storyteller-frontend"
-STORYTELLER_ML = "storyteller-ml"
-STORYTELLER_RUST = "/Users/bt/Development/storyteller-rust"
-```
-
-To download some ML models, run:
-
-```bash
-cargo run --bin download-job
-cargo run --bin tts-download-job
-```
-
-If you want to run both the HTTP API and the jobs, you'll need to run both processes in different
-terminals or tmux sessions.
-
-Note also that the configurations pointing to the ML monorepo must be set up for each application.
-Additionally, some development secrets may be needed (ask the team to share).
-
-To execute ML inference, run:
-
-```bash
-cargo run --bin inference-job
-cargo run --bin tts-inference-job
-```
-
-Again, note that the configurations pointing to the ML monorepo must be set up for each application.
-Additionally, some development secrets may be needed (ask the team to share).
-
-Seeding the Database with Some Values
--------------------------------------
-For voice designer you might want some pre-seeded audio that will hit the google cloud bucket, it will require env-secrets ask Brandon.
-cargo run --bin dev-database-seed -- --bucket
-
-Solutions to Common Setup Problems 
------------------------------------
-
-### [Fix] Sqlx Error when Performing Schema Migrations
-
-You might get this error message during migration,
-
-```
-Encountered unknown type for Mysql: enum
-thread 'main' panicked at 'internal error: entered unreachable code: Mysql only supports a closed set of types.
-                         If you ever see this error message please open an issue at https://github.com/diesel-rs/diesel 
-                         containing a dump of your schema definition.', diesel_cli/src/print_schema.rs:310:17
-note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
-```
-
-I haven't found the cause (it doesn't happen on newer installs), but the migrations appear to work regardless 
-of this error message. You can essentially ignore it.
-
-### [Fix] Can't connect to local MySQL after install on Ubuntu
-
-- If MySql in local dev can't be connected to, [reset the accounts](https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux).
-
-### API Server
-```
-cargo run --bin docs
-```
-This setups an api server with swagger docs to show you responses and requests with each associated endpoint
-It is run at http://127.0.0.1:8989/
-To obtain the json for the postman collection go to this url http://127.0.0.1:8989/api-docs/openapi.json
-We use this: `https://github.com/juhaku/utoipa`
-The documentation example is in: 
-`crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs`
-Write into the api doc to: 
-`crates/service/web/storyteller_web/src/api_doc.rs`

_sql/artcraft_migrations/20250708000000_create_tasks_table.sql

@@ -1,56 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- Migration history:
---   tasks_v1.sqlite - initial version
---   tasks_v2.sqlite - added model_type (nullable)
-
-CREATE TABLE tasks (
-    -- Task auto-incrementing primary key.
-    -- We don't use this.
-    id TEXT NOT NULL PRIMARY KEY,
-
-    -- TaskStatus enum
-    -- This is how the job system manages job states.
-    -- e.g. pending, started, complete_success, dead, etc.
-    task_status TEXT NOT NULL,
-
-    -- TaskType enum
-    -- e.g. image_generation, video_generation, etc.
-    task_type TEXT NOT NULL,
-
-    -- TaskModelType enum
-    -- e.g. flux_1_dev, veo_2, etc.
-    model_type TEXT,
-
-    -- GenerationProvider enum
-    -- Together with `provider_job_id`, this creates a foreign key lookup to the job.
-    -- e.g. 'artcraft', 'fal', 'sora', etc.
-    provider TEXT,
-
-    -- The primary key for the job in the provider's system.
-    provider_job_id TEXT,
-
-    -- OPTIONAL.
-    -- Tell the job system which caller initiated the task.
-    -- e.g. 'canvas'
-    frontend_caller TEXT,
-
-    -- OPTIONAL.
-    -- An arbitrary opaque identifier that frontend can set.
-    frontend_subscriber_id TEXT,
-
-    -- OPTIONAL.
-    -- An opaque JSON payload set by the frontend subscriber that
-    -- will be re-emitted back to the frontend.
-    frontend_subscriber_payload TEXT,
-
-    created_at INTEGER NOT NULL DEFAULT (unixepoch('now')),
-    updated_at INTEGER NOT NULL DEFAULT (unixepoch('now')),
-    completed_at INTEGER DEFAULT NULL
-);
-
--- Unique Indices
-CREATE UNIQUE INDEX idx_tasks_on_provider_job_id ON tasks(provider_job_id);
-
--- Indices
-CREATE INDEX idx_tasks_on_status ON tasks(task_status);
-CREATE INDEX idx_tasks_on_provider ON tasks(provider);

_sql/migrations/2025-06-02-085011_jobs_add_third_party_columns/down.sql

@@ -1,6 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-ALTER TABLE generic_inference_jobs DROP COLUMN maybe_external_third_party_id;
-ALTER TABLE generic_inference_jobs DROP COLUMN maybe_external_third_party;

_sql/migrations/2025-06-02-085011_jobs_add_third_party_columns/up.sql

@@ -1,9 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-ALTER TABLE generic_inference_jobs
-ADD COLUMN maybe_external_third_party VARCHAR(16) DEFAULT NULL;
-
-ALTER TABLE generic_inference_jobs
-ADD COLUMN maybe_external_third_party_id VARCHAR(64) DEFAULT NULL;

_sql/migrations/2025-06-02-085015_jobs_add_third_party_indices/down.sql

@@ -1,9 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-DROP INDEX `index_maybe_external_third_party_id`
-    ON `generic_inference_jobs`;
-
-DROP INDEX `index_maybe_external_third_party_and_id`
-    ON `generic_inference_jobs`;

_sql/migrations/2025-06-02-085015_jobs_add_third_party_indices/up.sql

@@ -1,11 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-ALTER TABLE generic_inference_jobs
-    ADD UNIQUE INDEX index_maybe_external_third_party_and_id (maybe_external_third_party, maybe_external_third_party_id),
-    ALGORITHM=INPLACE, LOCK=NONE;
-
-ALTER TABLE generic_inference_jobs
-    ADD INDEX index_maybe_external_third_party_id (maybe_external_third_party_id),
-    ALGORITHM=INPLACE, LOCK=NONE;

_sql/migrations/2025-07-23-031631_alter_prompts_add_model/down.sql

@@ -1,5 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-ALTER TABLE prompts DROP COLUMN maybe_model_type;

_sql/migrations/2025-07-23-031631_alter_prompts_add_model/up.sql

@@ -1,6 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-ALTER TABLE prompts
-    ADD COLUMN maybe_model_type VARCHAR(32) DEFAULT NULL;

_sql/migrations/2025-07-23-031650_alter_prompts_add_provider/down.sql

@@ -1,5 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-ALTER TABLE prompts DROP COLUMN maybe_generation_provider;

_sql/migrations/2025-07-23-031650_alter_prompts_add_provider/up.sql

@@ -1,6 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-ALTER TABLE prompts
-    ADD COLUMN maybe_generation_provider VARCHAR(16) DEFAULT NULL;

_sql/migrations/2025-07-23-031720_alter_prompts_add_maybe_anonymous_visitor_token/down.sql

@@ -1,5 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-ALTER TABLE prompts DROP COLUMN maybe_anonymous_visitor_token;

_sql/migrations/2025-07-23-031720_alter_prompts_add_maybe_anonymous_visitor_token/up.sql

@@ -1,6 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-ALTER TABLE prompts
-    ADD COLUMN maybe_anonymous_visitor_token VARCHAR(32) DEFAULT NULL;

_sql/migrations/2025-07-23-200905_prompt_context_items/down.sql

@@ -1,5 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-DROP TABLE prompt_context_items;

_sql/migrations/2025-07-23-200905_prompt_context_items/up.sql

@@ -1,30 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
--- An n:m join table between prompts and media_files.
--- This associates mostly image media files with prompts, though we could conceivably
--- also store 3D sets, etc.
-CREATE TABLE prompt_context_items (
-  -- Not used for anything except replication.
-  id BIGINT(20) NOT NULL AUTO_INCREMENT,
-
-  -- The semantic type of context.
-  context_semantic_type VARCHAR(16) NOT NULL,
-
-  -- Foreign key, many-to-many
-  -- There is no uniqueness constraint for (prompt_token, media_token) because a user
-  -- might attach many duplicate images to the same prompt.
-  prompt_token VARCHAR(32) NOT NULL,
-
-  -- Foreign key to the media token
-  -- There is no uniqueness constraint for (prompt_token, media_token) because a user
-  -- might attach many duplicate images to the same prompt.
-  media_token VARCHAR(32) NOT NULL,
-
-  -- INDICES --
-  PRIMARY KEY (id),
-  KEY index_prompt_token (prompt_token),
-  KEY index_media_token (media_token)
-
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_sql/migrations/2025-07-23-211516_alter_generic_inference_jobs_add_maybe_prompt_token/down.sql

@@ -1,5 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-ALTER TABLE generic_inference_jobs DROP COLUMN maybe_prompt_token;

_sql/migrations/2025-07-23-211516_alter_generic_inference_jobs_add_maybe_prompt_token/up.sql

@@ -1,6 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-ALTER TABLE generic_inference_jobs
-    ADD COLUMN maybe_prompt_token VARCHAR(32) DEFAULT NULL;

_sql/migrations/2025-07-23-212848_alter_media_files_add_maybe_generation_provider/down.sql

@@ -1,5 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-ALTER TABLE media_files DROP COLUMN maybe_generation_provider;

_sql/migrations/2025-07-23-212848_alter_media_files_add_maybe_generation_provider/up.sql

@@ -1,6 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
-ALTER TABLE media_files
-    ADD COLUMN maybe_generation_provider VARCHAR(16) DEFAULT NULL;

_sql/migrations_squashed/batch_generations.sql

@@ -15,9 +15,9 @@ CREATE TABLE batch_generations (
 
   -- The type of record this is attached to
   -- Currently supported values:
-  --   * 'model_weight' (deprecated)
-  --   * 'comment' (deprecated)
+  --   * 'model_weight'
   --   * 'media_file'
+  --   * 'comment'
   entity_type VARCHAR(32) NOT NULL,
 
   -- The token for the record.
@@ -37,4 +37,6 @@ CREATE TABLE batch_generations (
   -- NB: Realistically, we probably don't need this index until we have a lot of entity types, and the entity_token
   -- entropy is probably sufficient enough as-is.
   -- KEY index_entity_type_and_token (entity_type, entity_token),
+
+
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_sql/migrations_squashed/generic_inference_jobs.sql

@@ -21,8 +21,6 @@ CREATE TABLE generic_inference_jobs (
   -- The new enum for the type of job, which will eventually replace `inference_category` and `maybe_model_type`.
   -- This isn't fully supported yet in the inference-job, but we'll start populating it and adding logic around it.
   -- We'll start this out as nullable, then migrate the old rows with a default column value.
-  --
-  -- Fal jobs have the type `fal_queue`.
   job_type VARCHAR(32) DEFAULT NULL,
 
   -- There is an index on this column.
@@ -41,20 +39,8 @@ CREATE TABLE generic_inference_jobs (
   -- Examples (may not be up to date):
   --  * text_to_speech
   --  * voice_conversion
-  --  * image_generation
-  --  * video_generation  
-  --  * background_removal
   inference_category VARCHAR(32) NOT NULL,
 
-  -- If the job is externally enqueued, this is the third-party service
-  -- responsible for the job. Typically, we won't run an inference job for
-  -- jobs being run on a third party.
-  maybe_external_third_party VARCHAR(16) DEFAULT NULL,
-
-  -- If the job is externally enqueued, this is the ID the third-party
-  -- service uses.
-  maybe_external_third_party_id VARCHAR(64) DEFAULT NULL,
-
   -- There is an index on this column.
   -- NB: See notes on inference_category. This is becoming a problematic field. We're using
   -- this to load the container spin-up arguments and dependencies, but the former is being
@@ -105,11 +91,6 @@ CREATE TABLE generic_inference_jobs (
   -- And furthermore, not all download URLs have a cover image.
   maybe_cover_image_media_file_token VARCHAR(32) DEFAULT NULL,
 
-  -- If we pre-constructed a prompt, this will be copied to the resultant media generated.
-  -- This is especially important for Fal-related jobs and endpoints. We need this for keeping
-  -- track of all the prompt details, context images, etc.
-  maybe_prompt_token VARCHAR(30) DEFAULT NULL,
-
   -- A migration flag we can control from the backend API service which will tell
   -- the worker where to store the results. If false, GCP. If true, AWS.
   store_in_aws BOOLEAN NOT NULL DEFAULT FALSE,
@@ -306,13 +287,11 @@ CREATE TABLE generic_inference_jobs (
   PRIMARY KEY (id),
   UNIQUE KEY (token),
   UNIQUE KEY (uuid_idempotency_token),
-  UNIQUE KEY (maybe_external_third_party, maybe_external_third_party_id),
   KEY index_job_type (job_type),
   KEY index_product_category (product_category),
   KEY index_inference_category (inference_category),
   KEY index_maybe_model_type_and_maybe_model_token (maybe_model_type, maybe_model_token),
   KEY index_maybe_model_type (maybe_model_type),
-  KEY index_maybe_external_third_party_id (maybe_external_third_party_id),
   KEY fk_maybe_model_token (maybe_model_token),
   KEY fk_maybe_input_source_token (maybe_input_source_token),
   KEY fk_maybe_input_source_token_and_type (maybe_input_source_token, maybe_input_source_token_type),

_sql/migrations_squashed/media_files.sql

@@ -61,9 +61,6 @@ CREATE TABLE media_files (
   --   * 'workflow' (DEPRECATED)
   origin_product_category VARCHAR(16) NOT NULL DEFAULT "unknown",
 
-  -- What service provided the generation (if any).
-  maybe_generation_provider VARCHAR(16) DEFAULT NULL,
-
   -- For inference that can be tied back to a model, the type of model.
   -- There are multiple indices on this column!
   -- DO NOT EXPOSE THIS TO USERS VIA THE API, as we may leak secrets in doing so.

_sql/migrations_squashed/prompts.sql

@@ -19,25 +19,13 @@ CREATE TABLE prompts (
   -- Original product to which the prompt applied.
   -- Prompts may or may not be compatible across products.
   -- Possible values:
-  --  * 'artcraft_app'
-  --  * 'stable_diffusion' (legacy)
-  --  * 'comfy_ui' (legacy)
+  --  * 'stable_diffusion'
+  --  * 'comfy_ui'
   prompt_type VARCHAR(16) NOT NULL,
 
-  -- What type of model was used to generate the result.
-  -- NB: We're conflating prompts with outputs, but in a
-  -- sense the model is inextricably part of the prompt.
-  maybe_model_type VARCHAR(32) DEFAULT NULL,
-
-  -- What service provided the generation.
-  maybe_generation_provider VARCHAR(16) NOT NULL,
-
-  -- Whoever created the prompt
+  -- Whomever created the prompt
   maybe_creator_user_token VARCHAR(32) DEFAULT NULL,
 
-  -- Whoever created the prompt (anonymous user tracking)
-  maybe_anonymous_visitor_token VARCHAR(32) DEFAULT NULL,
-
   -- ========== PROMPT DETAILS ==========
 
   -- The positive prompt (optional)

_sql/migrations_squashed/prompts/prompt_context_items.sql

@@ -1,30 +0,0 @@
--- noinspection SqlDialectInspectionForFile
--- noinspection SqlNoDataSourceInspectionForFile
--- noinspection SqlResolveForFile
-
--- An n:m join table between prompts and media_files.
--- This associates mostly image media files with prompts, though we could conceivably
--- also store 3D sets, etc.
-CREATE TABLE prompt_context_items (
-  -- Not used for anything except replication.
-  id BIGINT(20) NOT NULL AUTO_INCREMENT,
-
-  -- Foreign key, many-to-many
-  -- There is no uniqueness constraint for (prompt_token, media_token) because a user
-  -- might attach many duplicate images to the same prompt.
-  prompt_token VARCHAR(32) NOT NULL,
-
-  -- Foreign key to the media token
-  -- There is no uniqueness constraint for (prompt_token, media_token) because a user
-  -- might attach many duplicate images to the same prompt.
-  media_token VARCHAR(32) NOT NULL,
-
-  -- The semantic type of context.
-  context_semantic_type VARCHAR(16) NOT NULL,
-
-  -- INDICES --
-  PRIMARY KEY (id),
-  KEY index_prompt_token (prompt_token),
-  KEY index_media_token (media_token)
-
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

crates/_types/composite_identifiers/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "composite_identifiers"
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+publish = false
+
+[lib]
+name = "composite_identifiers"
+path = "src/lib.rs"
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["workspace-build-acceleration"]
+
+[dependencies]
+# Internal
+enums = { path = "../enums" }
+tokens = { path = "../tokens" }
+
+# Workspace hack (faster compile times)
+workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
+
+# Serialization
+serde = "1.0.125"
+serde_derive = "1.0.125"
+
+sqlx = { version = "0.7.2", features = [ "mysql", "sqlite", "runtime-tokio-native-tls", "chrono" ] }
+sqlx-core = { version = "0.7.2" }
+sqlx-mysql= { version = "0.7.2" }
+
+utoipa = { version = "4", features = ["chrono","actix_extras"] }
+
+[dev-dependencies]
+
+# Test serialization
+serde_json = "1.0.108"

crates/_types/enums/Cargo.toml

@@ -0,0 +1,41 @@
+[package]
+name = "enums"
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+publish = false
+
+[lib]
+name = "enums"
+path = "src/lib.rs"
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["workspace-build-acceleration"]
+
+[dependencies]
+# Internal
+
+# Workspace hack (faster compile times)
+workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
+
+# Serialization
+serde = "1.0.125"
+serde_derive = "1.0.125"
+
+sqlx = { version = "0.7.2", features = [ "mysql", "sqlite", "runtime-tokio-native-tls", "chrono" ] }
+sqlx-core = { version = "0.7.2" }
+sqlx-mysql= { version = "0.7.2" }
+
+utoipa = { version = "4", features = ["chrono","actix_extras"] }
+
+[dev-dependencies]
+
+# Enum utilities
+strum = { version = "0.26", features = ["derive"] }
+
+# Test serialization
+serde_json = "1.0.108"
+toml = "0.8.14"