copy paste bug fixes

Actual backend integration pending

.github/workflows/artcraft-macos-publish.yml

@@ -0,0 +1,232 @@
+name: "Artcraft MacOS Publish"
+
+on:
+  push:
+    branches:
+      - release
+      - artcraft-release
+      - artcraft-macos  # Additional branch for MacOS testing
+
+# This workflow creates Universal macOS builds with signing certificate support.
+# On each push to the release branches it will create or update a GitHub release, build your app, and upload the artifacts to the release.
+
+jobs:
+  publish-artcraft:
+    permissions:
+      contents: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Universal macOS builds (arm64 and x86_64)
+          - platform: 'macos-latest-xlarge'
+            args: '--target universal-apple-darwin'
+          # Uncomment below for additional platforms
+          # - platform: 'ubuntu-22.04'
+          #   args: ''
+          # - platform: 'windows-latest'
+          #   args: ''
+
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: lts/*
+
+      - name: install Rust stable
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.platform == 'macos-latest-xlarge' && 'aarch64-apple-darwin,x86_64-apple-darwin' || '' }}
+
+      - name: install dependencies (ubuntu only)
+        if: matrix.platform == 'ubuntu-22.04'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf
+
+      - name: install frontend dependencies
+        run: |
+          # Navigate to the frontend directory to install dependencies
+          cd ./frontend
+
+          npm cache clean --force
+
+          echo "Installing all workspace dependencies..."
+          npm ci
+
+          echo "=== Version Information ==="
+          echo "Node version:"
+          node --version
+          echo "NPM version:"
+          npm --version
+
+          # Remove mismatched @nx packages and reinstall with correct versions
+          npm install --save-dev @nx/remix@21.1.1 @nx/js@21.1.1 @nx/react@21.1.1 @nx/vite@21.1.1 @nx/eslint@21.1.1
+
+          echo "=== Nx versions after adjustment ==="
+          npm list --depth=0 | grep nx
+
+      - name: import Apple Developer Certificate
+        if: matrix.platform == 'macos-latest-xlarge'
+        env:
+          APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
+          APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
+          KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
+        run: |
+          echo "Starting certificate import process..."
+
+          # Create certificate file
+          echo $APPLE_CERTIFICATE | base64 --decode > certificate.p12
+
+          # Debug: Check if certificate file exists and has content
+          echo "Certificate file info:"
+          ls -la certificate.p12
+          file certificate.p12
+
+          # Debug: Check certificate contents without password (just structure)
+          echo "Certificate structure (may show error if password protected):"
+          openssl pkcs12 -info -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -noout || echo "Could not read certificate structure"
+
+          # Debug: Try to list what's in the p12 file
+          echo "Attempting to list certificate contents:"
+          security cms -D -i certificate.p12 || echo "Not a CMS file, trying PKCS12..."
+          openssl pkcs12 -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -info -nodes 2>&1 | head -20 || echo "Failed to read PKCS12 contents"
+
+          # Debug: Extract certificate details to verify type and expiration
+          echo "Extracting certificate details..."
+          # Extract just the certificate portion and examine it
+          openssl pkcs12 -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -nokeys 2>/dev/null | openssl x509 -text -noout | grep -E "(Subject:|Issuer:|Not Before|Not After)" || echo "Failed to parse certificate details"
+
+          # Also try to see what type of certificate this is
+          echo "Certificate type check:"
+          openssl pkcs12 -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -nokeys 2>/dev/null | openssl x509 -text -noout | grep -E "(CN=|OU=)" || echo "Could not determine certificate type"
+
+          # Debug: Check if private key exists
+          echo "Checking for private key..."
+          openssl pkcs12 -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -nocerts -nodes 2>&1 | grep -E "(PRIVATE KEY|RSA KEY)" | head -5 || echo "No private key found"
+
+          # Create temporary keychain
+          security create-keychain -p "$KEYCHAIN_PASSWORD" build.keychain
+          security default-keychain -s build.keychain
+          security unlock-keychain -p "$KEYCHAIN_PASSWORD" build.keychain
+          security set-keychain-settings -t 3600 -u build.keychain
+
+          # Import certificate - ensure both certificate and private key are imported
+          echo "Importing certificate into keychain..."
+          # Import the p12 directly - this is the most reliable method
+          security import certificate.p12 -k build.keychain -P "$APPLE_CERTIFICATE_PASSWORD" -T /usr/bin/codesign -T /usr/bin/security -T /usr/bin/productbuild
+
+          # Alternative: Try with additional flags if the first import didn't create an identity
+          echo "Verifying import..."
+          if ! security find-identity -v build.keychain | grep -q "1 valid identities found"; then
+            echo "First import didn't create identity, trying with -A flag..."
+            security import certificate.p12 -k build.keychain -P "$APPLE_CERTIFICATE_PASSWORD" -A
+          fi
+
+          # Set trust settings for the certificate
+          echo "Setting trust settings..."
+          security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$KEYCHAIN_PASSWORD" build.keychain
+
+          # Unlock the keychain again to ensure it's accessible
+          security unlock-keychain -p "$KEYCHAIN_PASSWORD" build.keychain
+
+          # Add the keychain to the search list
+          security list-keychains -d user -s build.keychain $(security list-keychains -d user | sed 's/^[[:space:]]*"//;s/"$//')
+
+          # List all certificates to debug
+          echo "Listing all keychains:"
+          security list-keychains
+
+          echo "Listing all certificates in build keychain:"
+          security find-certificate -a -p build.keychain || echo "No certificates found"
+
+          echo "Detailed certificate info:"
+          security find-certificate -c "Learning Machines" build.keychain || echo "Certificate 'Learning Machines' not found"
+
+          # Check certificate trust settings
+          echo "Checking certificate trust settings:"
+          security dump-trust-settings || echo "No custom trust settings"
+
+          # Try to find codesigning identities
+          echo "Finding all identities in keychain:"
+          security find-identity -v build.keychain || echo "No identities found"
+
+          echo "Finding codesigning identities:"
+          security find-identity -v -p codesigning build.keychain || echo "No codesigning identities found"
+
+          # Additional debug: Check if this is a valid Developer ID cert
+          echo "Checking for Developer ID certificates:"
+          security find-certificate -a build.keychain | grep -A 5 -B 5 "Developer ID" || echo "No Developer ID certificates found"
+
+      - name: verify certificate
+        if: matrix.platform == 'macos-latest-xlarge'
+        run: |
+          echo "=== Certificate Verification ==="
+
+          echo "1. Checking codesigning identities:"
+          security find-identity -v -p codesigning build.keychain || echo "No codesigning identities found"
+
+          echo -e "\n2. Checking all identities:"
+          security find-identity -v build.keychain || echo "No identities found"
+
+          echo -e "\n3. Checking certificates in keychain:"
+          security find-certificate -c "Learning Machines" build.keychain -p | openssl x509 -text -noout | grep -E "(Subject:|CN=)" || echo "Could not parse certificate"
+
+          # Set the correct signing identity
+          echo "APPLE_SIGNING_IDENTITY=Developer ID Application: Learning Machines LLC (DJ6XS33FX8)" >> $GITHUB_ENV
+          echo "Proceeding with identity: Developer ID Application: Learning Machines LLC (DJ6XS33FX8)"
+
+          # Also set the team ID
+          echo "APPLE_DEVELOPMENT_TEAM=DJ6XS33FX8" >> $GITHUB_ENV
+
+      - name: Pre-build frontend to avoid Tauri build issues
+        run: |
+          # Build the frontend app from the correct directory
+          cd ./frontend
+
+          # Increase Node.js memory limit for the build
+          export NODE_OPTIONS="--max-old-space-size=8192"
+
+          # Use npx to ensure we use the local nx version
+          echo "Building with npx nx..."
+          npx nx run editor3d:build
+
+          echo "Checking build output..."
+          ls -la apps/editor3d/dist/ | head -10
+
+          # Update Tauri config to skip the build (since we already built)
+          cd ${{ github.workspace }}/crates/desktop/artcraft
+          cp tauri.conf.json tauri.conf.json.bak
+          # Set beforeBuildCommand to empty or a no-op command
+          if command -v jq &> /dev/null; then
+            jq '.build.beforeBuildCommand = ""' tauri.conf.json > tauri.conf.json.tmp
+            mv tauri.conf.json.tmp tauri.conf.json
+          else
+            node -e "const fs = require('fs'); const config = JSON.parse(fs.readFileSync('tauri.conf.json', 'utf8')); config.build.beforeBuildCommand = ''; fs.writeFileSync('tauri.conf.json', JSON.stringify(config, null, 2));"
+          fi
+
+
+      - name: build and publish
+        uses: tauri-apps/tauri-action@v0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          APPLE_ID: ${{ secrets.APPLE_ID }}
+          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
+          APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
+          APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
+          APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
+          APPLE_SIGNING_IDENTITY: ${{ env.APPLE_SIGNING_IDENTITY }}
+          APPLE_DEVELOPMENT_TEAM: ${{ env.APPLE_DEVELOPMENT_TEAM }}
+          # Use the SQLx query cache for SQLite
+          SQLX_OFFLINE: true
+        with:
+          projectPath: ./crates/desktop/artcraft
+          tagName: artcraft-v__VERSION__ # the action automatically replaces __VERSION__ with the app version.
+          releaseName: "Artcraft v__VERSION__"
+          releaseBody: "See the assets to download this version and install."
+          releaseDraft: true
+          prerelease: false
+          args: ${{ matrix.args }}

.github/workflows/artcraft-publish.yml

@@ -1,107 +0,0 @@
-name: "Artcraft Publish"
-
-on:
-  push:
-    branches:
-      - release
-      - artcraft-release
-
-# This workflow creates Universal macOS builds with signing certificate support.
-# On each push to the release branches it will create or update a GitHub release, build your app, and upload the artifacts to the release.
-
-jobs:
-  publish-artcraft:
-    permissions:
-      contents: write
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # Universal macOS builds (arm64 and x86_64)
-          - platform: 'macos-latest'
-            args: '--target universal-apple-darwin'
-          # Uncomment below for additional platforms
-          # - platform: 'ubuntu-22.04'
-          #   args: ''
-          # - platform: 'windows-latest'
-          #   args: ''
-
-    runs-on: ${{ matrix.platform }}
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: setup node
-        uses: actions/setup-node@v4
-        with:
-          node-version: lts/*
-
-      - name: install Rust stable
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          targets: ${{ matrix.platform == 'macos-latest' && 'aarch64-apple-darwin,x86_64-apple-darwin' || '' }}
-
-      - name: install dependencies (ubuntu only)
-        if: matrix.platform == 'ubuntu-22.04'
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf
-
-      - name: install frontend dependencies
-        run: |
-          cd ./crates/desktop/artcraft
-          yarn install
-
-      - name: import Apple Developer Certificate
-        if: matrix.platform == 'macos-latest'
-        # Prevents keychain from locking automatically for 3600 seconds.
-        env:
-          APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
-          APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
-          KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
-        run: |
-          echo $APPLE_CERTIFICATE | base64 --decode > certificate.p12
-          security create-keychain -p "$KEYCHAIN_PASSWORD" build.keychain
-          security default-keychain -s build.keychain
-          security unlock-keychain -p "$KEYCHAIN_PASSWORD" build.keychain
-          security set-keychain-settings -t 3600 -u build.keychain
-          security import certificate.p12 -k build.keychain -P "$APPLE_CERTIFICATE_PASSWORD" -T /usr/bin/codesign
-          security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$KEYCHAIN_PASSWORD" build.keychain
-          security find-identity -v -p codesigning build.keychain
-
-      - name: verify certificate
-        if: matrix.platform == 'macos-latest'
-        run: |
-          echo "Available certificates:"
-          security find-identity -v -p codesigning build.keychain
-          
-          # Try multiple certificate patterns
-          CERT_INFO=$(security find-identity -v -p codesigning build.keychain | grep -E "(Developer ID Application|Apple Development|Storyteller Developer)" | head -1)
-          
-          if [ -z "$CERT_INFO" ]; then
-            echo "No matching certificate found. Using first available certificate."
-            CERT_INFO=$(security find-identity -v -p codesigning build.keychain | grep -v "Policy" | head -1)
-          fi
-          
-          CERT_ID=$(echo "$CERT_INFO" | awk -F'"' '{print $2}')
-          echo "CERT_ID=$CERT_ID" >> $GITHUB_ENV
-          echo "Using certificate: $CERT_ID"
-
-      - name: build and publish
-        uses: tauri-apps/tauri-action@v0
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          APPLE_ID: ${{ secrets.APPLE_ID }}
-          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
-          APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
-          APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
-          APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
-          APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
-          APPLE_SIGNING_IDENTITY: ${{ env.CERT_ID }}
-        with:
-          projectPath: ./crates/desktop/artcraft
-          tagName: artcraft-v__VERSION__ # the action automatically replaces __VERSION__ with the app version.
-          releaseName: "Artcraft v__VERSION__"
-          releaseBody: "See the assets to download this version and install."
-          releaseDraft: true
-          prerelease: false
-          args: ${{ matrix.args }}

.github/workflows/artcraft-windows-publish.yml

@@ -0,0 +1,96 @@
+name: "Artcraft Windows Publish"
+
+on:
+  push:
+    branches:
+      - release
+      - artcraft-release
+      - artcraft-windows  # Additional branch for Windows testing
+
+# This workflow creates Windows builds.
+# On each push to the release branches it will create or update a GitHub release, build your app, and upload the artifacts to the release.
+
+jobs:
+  publish-artcraft-windows:
+    permissions:
+      contents: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Windows builds
+          - platform: 'windows-latest'
+            args: ''
+
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: lts/*
+
+      - name: install Rust stable
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: install frontend dependencies
+        run: |
+          # Navigate to the frontend directory to install dependencies
+          cd ./frontend
+
+          npm cache clean --force
+
+          echo "Installing all workspace dependencies..."
+          npm ci
+
+          echo "=== Version Information ==="
+          echo "Node version:"
+          node --version
+          echo "NPM version:"
+          npm --version
+
+          # Remove mismatched @nx packages and reinstall with correct versions
+          npm install --save-dev @nx/remix@21.1.1 @nx/js@21.1.1 @nx/react@21.1.1 @nx/vite@21.1.1 @nx/eslint@21.1.1
+
+          echo "=== Nx versions after adjustment ==="
+          npm list --depth=0 | findstr nx
+
+      - name: Pre-build frontend to avoid Tauri build issues
+        run: |
+          # Build the frontend app from the correct directory
+          cd ./frontend
+
+          # Increase Node.js memory limit for the build
+          $env:NODE_OPTIONS="--max-old-space-size=8192"
+
+          # Use npx to ensure we use the local nx version
+          echo "Building with npx nx..."
+          npx nx run editor3d:build
+
+          echo "Checking build output..."
+          dir apps/editor3d/dist/ | Select-Object -First 10
+
+          # Update Tauri config to skip the build (since we already built)
+          cd ${{ github.workspace }}/crates/desktop/artcraft
+          Copy-Item tauri.conf.json tauri.conf.json.bak
+
+          # Set beforeBuildCommand to empty using PowerShell
+          $config = Get-Content tauri.conf.json | ConvertFrom-Json
+          $config.build.beforeBuildCommand = ""
+          $config | ConvertTo-Json -Depth 10 | Set-Content tauri.conf.json
+
+      - name: build and publish
+        uses: tauri-apps/tauri-action@v0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # Use the SQLx query cache for SQLite
+          SQLX_OFFLINE: true
+        with:
+          projectPath: ./crates/desktop/artcraft
+          tagName: artcraft-windows-v__VERSION__ # the action automatically replaces __VERSION__ with the app version.
+          releaseName: "Artcraft Windows v__VERSION__"
+          releaseBody: "Windows build - See the assets to download this version and install."
+          releaseDraft: true
+          prerelease: false
+          args: ${{ matrix.args }}

.gitignore

@@ -24,11 +24,13 @@ twitch_secrets.toml
 *-secrets.env
 
 # Utilities with secrets
-script/production_database_secret_backup.sh
-script/production_database_secret_connect.sh
 production_database_secret_connect.sh
 production_database_secret_query.sh
 production_database_voice_clone_report_download.sh
+script/artcraft/mac_signing_build.sh
+script/production_database_secret_backup.sh
+script/production_database_secret_connect.sh
+script/socat_dev_database_connect.sh
 
 # Scratch data
 scratch/

.sqlx/query-002cdb3d086f3b4181e2997befa54bd8516cb1d3d3bb6e389862695318d5c299.json

@@ -1,12 +1,12 @@
 {
   "db_name": "MySQL",
-  "query": "\nINSERT INTO generic_inference_jobs\nSET\n  token = ?,\n  uuid_idempotency_token = ?,\n\n  job_type = ?,\n\n  maybe_external_third_party = ?,\n  maybe_external_third_party_id = ?,\n\n  product_category = ?,\n  inference_category = ?,\n\n  maybe_model_type = NULL,\n  maybe_model_token = NULL,\n\n  maybe_input_source_token = NULL,\n  maybe_input_source_token_type = NULL,\n\n  maybe_download_url = NULL,\n  maybe_cover_image_media_file_token = NULL,\n\n  maybe_raw_inference_text = NULL,\n\n  maybe_inference_args = ?,\n\n  maybe_creator_user_token = ?,\n  maybe_creator_anonymous_visitor_token = ?,\n  creator_ip_address = ?,\n  creator_set_visibility = ?,\n\n  priority_level = 0,\n  is_keepalive_required = FALSE,\n  max_duration_seconds = 0,\n\n  is_debug_request = FALSE,\n  maybe_routing_tag = NULL,\n\n  status = ?\n        ",
+  "query": "\nINSERT INTO generic_inference_jobs\nSET\n  token = ?,\n  uuid_idempotency_token = ?,\n\n  job_type = ?,\n\n  maybe_external_third_party = ?,\n  maybe_external_third_party_id = ?,\n\n  product_category = ?,\n  inference_category = ?,\n\n  maybe_model_type = NULL,\n  maybe_model_token = NULL,\n\n  maybe_input_source_token = NULL,\n  maybe_input_source_token_type = NULL,\n\n  maybe_download_url = NULL,\n  maybe_cover_image_media_file_token = NULL,\n  \n  maybe_prompt_token = ?,\n\n  maybe_raw_inference_text = NULL,\n\n  maybe_inference_args = ?,\n\n  maybe_creator_user_token = ?,\n  maybe_creator_anonymous_visitor_token = ?,\n  creator_ip_address = ?,\n  creator_set_visibility = ?,\n\n  priority_level = 0,\n  is_keepalive_required = FALSE,\n  max_duration_seconds = 0,\n\n  is_debug_request = FALSE,\n  maybe_routing_tag = NULL,\n\n  status = ?\n        ",
   "describe": {
     "columns": [],
     "parameters": {
-      "Right": 13
+      "Right": 14
     },
     "nullable": []
   },
-  "hash": "0fa824822cd94024dce21243f41e69e4a3e8685c4de676834a7c0ee14982c976"
+  "hash": "002cdb3d086f3b4181e2997befa54bd8516cb1d3d3bb6e389862695318d5c299"
 }

.sqlx/query-01a0bf33e8331c19dc73f064c54eaa863bec158ef7713450a06c75e246f5f08c.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "SQLite",
+  "query": "\n    INSERT INTO tasks (\n      id,\n      task_status,\n      task_type,\n      model_type,\n      provider,\n      provider_job_id,\n      frontend_caller,\n      frontend_subscriber_id,\n      frontend_subscriber_payload\n    )\n    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)\n  ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 9
+    },
+    "nullable": []
+  },
+  "hash": "01a0bf33e8331c19dc73f064c54eaa863bec158ef7713450a06c75e246f5f08c"
+}

.sqlx/query-08f52f15acf6afc7091a525e3dc88f706c71c3579e5e81aa9903807d85f469ef.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\nINSERT INTO prompts\nSET\n  token = ?,\n  prompt_type = ?,\n\n  maybe_creator_user_token = ?,\n  \n  maybe_model_type = ?,\n  maybe_generation_provider = ?,\n\n  maybe_positive_prompt = ?,\n  maybe_negative_prompt = ?,\n  \n  maybe_other_args = ?,\n\n  creator_ip_address = ?\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 9
+    },
+    "nullable": []
+  },
+  "hash": "08f52f15acf6afc7091a525e3dc88f706c71c3579e5e81aa9903807d85f469ef"
+}

.sqlx/query-094eb099cde85835db0662c2700c6b82644b51bf9b3609391ed4d2945fe5e98f.json

@@ -0,0 +1,69 @@
+{
+  "db_name": "MySQL",
+  "query": "\nSELECT\n    pci.media_token as `media_token: tokens::tokens::media_files::MediaFileToken`,\n    pci.context_semantic_type as `context_semantic_type: enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType`,\n\n    m.public_bucket_directory_hash,\n    m.maybe_public_bucket_prefix,\n    m.maybe_public_bucket_extension\n\nFROM prompt_context_items pci\nJOIN media_files m\nON\n    pci.media_token = m.token\nWHERE\n    pci.prompt_token = ?\nORDER BY pci.id ASC\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "media_token: tokens::tokens::media_files::MediaFileToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 1,
+        "name": "context_semantic_type: enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 64
+        }
+      },
+      {
+        "ordinal": 2,
+        "name": "public_bucket_directory_hash",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 3,
+        "name": "maybe_public_bucket_prefix",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 64
+        }
+      },
+      {
+        "ordinal": 4,
+        "name": "maybe_public_bucket_extension",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 64
+        }
+      }
+    ],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      true,
+      true
+    ]
+  },
+  "hash": "094eb099cde85835db0662c2700c6b82644b51bf9b3609391ed4d2945fe5e98f"
+}

.sqlx/query-13ca06bcbb4a576f8a2229a8554b7488d379baa381d8d5cb60972750e81b2173.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "MySQL",
-  "query": "\nINSERT INTO prompts\nSET\n  token = ?,\n  prompt_type = ?,\n\n  maybe_creator_user_token = ?,\n\n  maybe_positive_prompt = ?,\n  maybe_negative_prompt = ?,\n  maybe_other_args = ?,\n\n  creator_ip_address = ?\n        ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 7
-    },
-    "nullable": []
-  },
-  "hash": "13ca06bcbb4a576f8a2229a8554b7488d379baa381d8d5cb60972750e81b2173"
-}

.sqlx/query-19708db99f2b3c6283d2115a511e0eb1c99fab396ad0effd9eb2dd79926e9c93.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\nUPDATE generic_inference_jobs\nSET\n  status = 'cancelled_by_system',\n  retry_at = NULL,\n  internal_debugging_failure_reason = 'reap_stale_fakeyou_jobs'\nWHERE\n  job_type != 'fal_queue'\n  AND status IN (\n    'pending',\n    'started',\n    'attempt_failed'\n   )\n   AND created_at > NOW() - INTERVAL 30 MINUTE\nLIMIT 5000\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 0
+    },
+    "nullable": []
+  },
+  "hash": "19708db99f2b3c6283d2115a511e0eb1c99fab396ad0effd9eb2dd79926e9c93"
+}

.sqlx/query-b46b0d022790f4e10f6e47126df0fdc55172bf8cda77d4e0b2afac9e6ebd1630.json

@@ -1,6 +1,6 @@
 {
   "db_name": "MySQL",
-  "query": "\nSELECT\n    p.token as `token: tokens::tokens::prompts::PromptToken`,\n\n    p.prompt_type as `prompt_type: enums::by_table::prompts::prompt_type::PromptType`,\n\n    p.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n\n    p.maybe_positive_prompt,\n    p.maybe_negative_prompt,\n    p.maybe_other_args,\n\n    p.creator_ip_address,\n    p.created_at\n\nFROM prompts as p\nWHERE\n    p.token = ?\n        ",
+  "query": "\nSELECT\n    p.token as `token: tokens::tokens::prompts::PromptToken`,\n\n    p.prompt_type as `prompt_type: enums::by_table::prompts::prompt_type::PromptType`,\n\n    p.maybe_model_type as `maybe_model_type: enums::common::model_type::ModelType`,\n    p.maybe_generation_provider as `maybe_generation_provider: enums::common::generation_provider::GenerationProvider`,\n\n    p.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n\n    p.maybe_positive_prompt,\n    p.maybe_negative_prompt,\n    p.maybe_other_args,\n\n    p.creator_ip_address,\n    p.created_at\n\nFROM prompts as p\nWHERE\n    p.token = ?\n        ",
   "describe": {
     "columns": [
       {
@@ -25,6 +25,26 @@
       },
       {
         "ordinal": 2,
+        "name": "maybe_model_type: enums::common::model_type::ModelType",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 3,
+        "name": "maybe_generation_provider: enums::common::generation_provider::GenerationProvider",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 64
+        }
+      },
+      {
+        "ordinal": 4,
         "name": "maybe_creator_user_token: tokens::tokens::users::UserToken",
         "type_info": {
           "type": "VarString",
@@ -34,7 +54,7 @@
         }
       },
       {
-        "ordinal": 3,
+        "ordinal": 5,
         "name": "maybe_positive_prompt",
         "type_info": {
           "type": "Blob",
@@ -44,7 +64,7 @@
         }
       },
       {
-        "ordinal": 4,
+        "ordinal": 6,
         "name": "maybe_negative_prompt",
         "type_info": {
           "type": "Blob",
@@ -54,7 +74,7 @@
         }
       },
       {
-        "ordinal": 5,
+        "ordinal": 7,
         "name": "maybe_other_args",
         "type_info": {
           "type": "Blob",
@@ -64,7 +84,7 @@
         }
       },
       {
-        "ordinal": 6,
+        "ordinal": 8,
         "name": "creator_ip_address",
         "type_info": {
           "type": "VarString",
@@ -74,7 +94,7 @@
         }
       },
       {
-        "ordinal": 7,
+        "ordinal": 9,
         "name": "created_at",
         "type_info": {
           "type": "Timestamp",
@@ -94,9 +114,11 @@
       true,
       true,
       true,
+      true,
+      true,
       false,
       false
     ]
   },
-  "hash": "61744bbf83f7cf00348bcc21e868d9c506fec46da4a33ac50d63c6d9e00c0cec"
+  "hash": "b46b0d022790f4e10f6e47126df0fdc55172bf8cda77d4e0b2afac9e6ebd1630"
 }

.sqlx/query-dc0367e9e43c3d7dda3623e77337ab4529eac2192475be7d08589886703ba1cb.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "SQLite",
+  "query": "\n    UPDATE tasks\n    SET task_status = ?\n    WHERE id = ?\n  ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 2
+    },
+    "nullable": []
+  },
+  "hash": "dc0367e9e43c3d7dda3623e77337ab4529eac2192475be7d08589886703ba1cb"
+}

.sqlx/query-fbe482c8987231c3b5d75f2e792d0fdd033de8a7eb821428fa034752c73edfb7.json

@@ -1,6 +1,6 @@
 {
   "db_name": "MySQL",
-  "query": "\nSELECT\n    jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,\n\n    jobs.status as `status: enums::common::job_status_plus::JobStatusPlus`,\n    \n    jobs.maybe_external_third_party as `external_third_party: enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty`,\n    jobs.maybe_external_third_party_id as `external_third_party_id`,\n\n    jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n    jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,\n    jobs.creator_ip_address,\n\n    jobs.created_at,\n    jobs.updated_at\n\nFROM generic_inference_jobs as jobs\n\nWHERE jobs.maybe_external_third_party = ?\nAND jobs.maybe_external_third_party_id = ?\n        ",
+  "query": "\nSELECT\n    jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,\n\n    jobs.status as `status: enums::common::job_status_plus::JobStatusPlus`,\n    \n    jobs.maybe_external_third_party as `external_third_party: enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty`,\n    jobs.maybe_external_third_party_id as `external_third_party_id`,\n\n    jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n    jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,\n    jobs.creator_ip_address,\n    \n    jobs.maybe_prompt_token as `maybe_prompt_token: tokens::tokens::prompts::PromptToken`,\n\n    jobs.created_at,\n    jobs.updated_at\n\nFROM generic_inference_jobs as jobs\n\nWHERE jobs.maybe_external_third_party = ?\nAND jobs.maybe_external_third_party_id = ?\n        ",
   "describe": {
     "columns": [
       {
@@ -75,6 +75,16 @@
       },
       {
         "ordinal": 7,
+        "name": "maybe_prompt_token: tokens::tokens::prompts::PromptToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 8,
         "name": "created_at",
         "type_info": {
           "type": "Timestamp",
@@ -84,7 +94,7 @@
         }
       },
       {
-        "ordinal": 8,
+        "ordinal": 9,
         "name": "updated_at",
         "type_info": {
           "type": "Timestamp",
@@ -105,9 +115,10 @@
       true,
       true,
       false,
+      true,
       false,
       false
     ]
   },
-  "hash": "64dfacc1f0552d539f009fea91e902a2fe296c06ad539808596bbf2c8fc9eaa0"
+  "hash": "fbe482c8987231c3b5d75f2e792d0fdd033de8a7eb821428fa034752c73edfb7"
 }

Cargo.toml

@@ -4,6 +4,18 @@ resolver = "2"
 # This block defines the workspace members.
 # When we add a library or binary crate, it needs an entry here.
 members = [
+  # API clients for external services
+  # These might be custom implementations or wrappers over other client libraries.
+  "crates/api_clients/cloud_storage",
+  "crates/api_clients/fal_client",
+  "crates/api_clients/google_drive_common",
+  "crates/api_clients/google_sign_in",
+  "crates/api_clients/openai_sora_client",
+  "crates/api_clients/openart_client",
+  "crates/api_clients/runwayml_client",
+  "crates/api_clients/storyteller_client",
+  "crates/api_clients/thumbnail_generator",
+
   # CLI tools for operations, etc.
   "crates/cli/devenv/dev-database-seed",
   "crates/cli/devenv/dev-upload-media-file",
@@ -22,21 +34,13 @@ members = [
   # Shared libraries
   "crates/lib/actix_cors_configs",
   "crates/lib/actix_helpers",
-  "crates/lib/api_clients/cloud_storage",
-  "crates/lib/api_clients/fal_client",
-  "crates/lib/api_clients/google_drive_common",
-  "crates/lib/api_clients/google_sign_in",
-  "crates/lib/api_clients/openai_sora_client",
-  "crates/lib/api_clients/openart_client",
-  "crates/lib/api_clients/runwayml_client",
-  "crates/lib/api_clients/storyteller_client",
-  "crates/lib/api_clients/thumbnail_generator",
   "crates/lib/bootstrap",
   "crates/lib/bucket_paths",
   "crates/lib/build_metadata",
   "crates/lib/caching/memory_caching",
   "crates/lib/caching/memory_store",
   "crates/lib/caching/redis_caching",
+  "crates/lib/cloudflare_errors",
   "crates/lib/concurrency",
   "crates/lib/cookies",
   "crates/lib/data/collections",
@@ -73,6 +77,7 @@ members = [
   "crates/lib/sqlx_mysql_helpers",
   "crates/lib/storyteller_root",
   "crates/lib/subprocess_common",
+  "crates/lib/test_data",
   "crates/lib/testing",
   "crates/lib/tts_common",
   "crates/lib/url_config",
@@ -86,6 +91,7 @@ members = [
   "crates/schema/database/mysql_queries",
   "crates/schema/database/redis_common",
   "crates/schema/database/redis_schema",
+  "crates/schema/database/sqlite_tasks",
   #"crates/schema/database/sqlite_queries",
 
   # Schema (public): common strongly-typed tokens, ids, payloads, etc.
@@ -210,11 +216,13 @@ url = { version = "2.5.4", features = ["serde"] }
 # Internal libs
 artcraft_api_defs = { path = "crates/schema/public/api/artcraft_api_defs" }
 build_metadata = { path = "crates/lib/build_metadata" }
+cloud_storage = { path = "crates/api_clients/cloud_storage" }
+cloudflare_errors = { path = "crates/lib/cloudflare_errors" }
 crockford = { path = "crates/lib/data/crockford" }
 enums = { path = "crates/schema/public/enums" }
 enums_public = { path = "crates/schema/public/enums_public" }
 errors = { path = "crates/lib/errors" }
-fal_client = { path = "crates/lib/api_clients/fal_client" }
+fal_client = { path = "crates/api_clients/fal_client" }
 file_to_base64_url = { path = "crates/lib/files/file_to_base64_url" }
 filesys = { path = "crates/lib/files/filesys" }
 idempotency = { path = "crates/lib/idempotency" }
@@ -223,10 +231,12 @@ jwt_light = { path = "crates/lib/jwt_light" }
 memory_caching = { path = "crates/lib/caching/memory_caching" }
 memory_store = { path = "crates/lib/caching/memory_store" }
 mimetypes = { path = "crates/lib/files/mimetypes" }
-openai_sora_client = { path = "crates/lib/api_clients/openai_sora_client" }
+openai_sora_client = { path = "crates/api_clients/openai_sora_client" }
 primitives = { path = "crates/lib/data/primitives" }
 shared_service_components = { path = "crates/lib/shared_service_components" }
-storyteller_client = { path = "crates/lib/api_clients/storyteller_client" }
+sqlite_tasks = { path = "crates/schema/database/sqlite_tasks" }
+storyteller_client = { path = "crates/api_clients/storyteller_client" }
 subprocess_common = { path = "crates/lib/subprocess_common" }
+test_data = { path = "crates/lib/test_data" }
 tokens = { path = "crates/schema/public/tokens" }
-web_base64= { path = "crates/lib/data/web_base64" }
+web_base64 = { path = "crates/lib/data/web_base64" }

README.md

@@ -1,7 +1,7 @@
 storyteller-rust
 ================
 
-Our Rust monorepo, which serves both our API surface area and our downloadable client applications.
+Our Artcraft monorepo, which serves our desktop application, backend infrastructure, and more.
 
 ```
                  ."-,.__
@@ -45,5 +45,6 @@ Our Rust monorepo, which serves both our API surface area and our downloadable c
                  / ,"'"\,'               `/  `-.|" 
 ```
 
-- [documentation](./_develop/docs)
-- [tools, misc](./_develop)
+- [developer documentation](./_docs)
+- [server API reference](https://storyteller-docs.netlify.app/)
+- [tools, scripts, misc](./script)

_docs/code_conventions.md

@@ -0,0 +1,26 @@
+Code Conventions
+================
+
+We use three languages at Artcraft: Rust, TypeScript, and Python.
+
+## Rust style guide
+
+We deviate from the official Rust style guide in a few places:
+
+- We use two spaces for indentation instead of four.
+- We typically keep one function or type definition per file, though this isn't a steadfast rule.
+- We use a Cargo workspace to share code within our monorepo.
+
+### Rust Preferred Libraries
+
+We use the following Rust libraries, though we are open to migrating to alternatives:
+
+- `actix-web` for our HTTP web server
+- `reqwest` for HTTP client requests
+- `sqlx` for SQL database operations
+- `tokio` for asynchronous programming
+
+## TypeScript style guide
+
+- We use two spaces for indentation instead of four.
+- We use `nx` for monorepo management

_docs/local_development_setup_guide.md

@@ -1,5 +1,5 @@
-FakeYou Development Setup (bare metal)
-======================================
+Local Development Setup Guide (for bare metal)
+==============================================
 
 This document describes how to run the FakeYou infrastructure on your development 
 machine's bare metal (no containers).

_sql/artcraft_migrations/20250708000000_create_tasks_table.sql

@@ -0,0 +1,56 @@
+-- noinspection SqlDialectInspectionForFile
+-- Migration history:
+--   tasks_v1.sqlite - initial version
+--   tasks_v2.sqlite - added model_type (nullable)
+
+CREATE TABLE tasks (
+    -- Task auto-incrementing primary key.
+    -- We don't use this.
+    id TEXT NOT NULL PRIMARY KEY,
+
+    -- TaskStatus enum
+    -- This is how the job system manages job states.
+    -- e.g. pending, started, complete_success, dead, etc.
+    task_status TEXT NOT NULL,
+
+    -- TaskType enum
+    -- e.g. image_generation, video_generation, etc.
+    task_type TEXT NOT NULL,
+
+    -- TaskModelType enum
+    -- e.g. flux_1_dev, veo_2, etc.
+    model_type TEXT,
+
+    -- GenerationProvider enum
+    -- Together with `provider_job_id`, this creates a foreign key lookup to the job.
+    -- e.g. 'artcraft', 'fal', 'sora', etc.
+    provider TEXT,
+
+    -- The primary key for the job in the provider's system.
+    provider_job_id TEXT,
+
+    -- OPTIONAL.
+    -- Tell the job system which caller initiated the task.
+    -- e.g. 'canvas'
+    frontend_caller TEXT,
+
+    -- OPTIONAL.
+    -- An arbitrary opaque identifier that frontend can set.
+    frontend_subscriber_id TEXT,
+
+    -- OPTIONAL.
+    -- An opaque JSON payload set by the frontend subscriber that
+    -- will be re-emitted back to the frontend.
+    frontend_subscriber_payload TEXT,
+
+    created_at INTEGER NOT NULL DEFAULT (unixepoch('now')),
+    updated_at INTEGER NOT NULL DEFAULT (unixepoch('now')),
+    completed_at INTEGER DEFAULT NULL
+);
+
+-- Unique Indices
+CREATE UNIQUE INDEX idx_tasks_on_provider_job_id ON tasks(provider_job_id);
+
+-- Indices
+CREATE INDEX idx_tasks_on_status ON tasks(task_status);
+CREATE INDEX idx_tasks_on_provider ON tasks(provider);

_sql/migrations/2025-07-23-031631_alter_prompts_add_model/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE prompts DROP COLUMN maybe_model_type;

_sql/migrations/2025-07-23-031631_alter_prompts_add_model/up.sql

@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE prompts
+    ADD COLUMN maybe_model_type VARCHAR(32) DEFAULT NULL;

_sql/migrations/2025-07-23-031650_alter_prompts_add_provider/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE prompts DROP COLUMN maybe_generation_provider;

_sql/migrations/2025-07-23-031650_alter_prompts_add_provider/up.sql

@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE prompts
+    ADD COLUMN maybe_generation_provider VARCHAR(16) DEFAULT NULL;

_sql/migrations/2025-07-23-031720_alter_prompts_add_maybe_anonymous_visitor_token/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE prompts DROP COLUMN maybe_anonymous_visitor_token;

_sql/migrations/2025-07-23-031720_alter_prompts_add_maybe_anonymous_visitor_token/up.sql

@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE prompts
+    ADD COLUMN maybe_anonymous_visitor_token VARCHAR(32) DEFAULT NULL;

_sql/migrations/2025-07-23-200905_prompt_context_items/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+DROP TABLE prompt_context_items;

_sql/migrations/2025-07-23-200905_prompt_context_items/up.sql

@@ -0,0 +1,30 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- An n:m join table between prompts and media_files.
+-- This associates mostly image media files with prompts, though we could conceivably
+-- also store 3D sets, etc.
+CREATE TABLE prompt_context_items (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- The semantic type of context.
+  context_semantic_type VARCHAR(16) NOT NULL,
+
+  -- Foreign key, many-to-many
+  -- There is no uniqueness constraint for (prompt_token, media_token) because a user
+  -- might attach many duplicate images to the same prompt.
+  prompt_token VARCHAR(32) NOT NULL,
+
+  -- Foreign key to the media token
+  -- There is no uniqueness constraint for (prompt_token, media_token) because a user
+  -- might attach many duplicate images to the same prompt.
+  media_token VARCHAR(32) NOT NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  KEY index_prompt_token (prompt_token),
+  KEY index_media_token (media_token)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_sql/migrations/2025-07-23-211516_alter_generic_inference_jobs_add_maybe_prompt_token/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE generic_inference_jobs DROP COLUMN maybe_prompt_token;

_sql/migrations/2025-07-23-211516_alter_generic_inference_jobs_add_maybe_prompt_token/up.sql

@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE generic_inference_jobs
+    ADD COLUMN maybe_prompt_token VARCHAR(32) DEFAULT NULL;

_sql/migrations/2025-07-23-212848_alter_media_files_add_maybe_generation_provider/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE media_files DROP COLUMN maybe_generation_provider;

_sql/migrations/2025-07-23-212848_alter_media_files_add_maybe_generation_provider/up.sql

@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE media_files
+    ADD COLUMN maybe_generation_provider VARCHAR(16) DEFAULT NULL;

_sql/migrations_squashed/generic_inference_jobs.sql

@@ -21,6 +21,8 @@ CREATE TABLE generic_inference_jobs (
   -- The new enum for the type of job, which will eventually replace `inference_category` and `maybe_model_type`.
   -- This isn't fully supported yet in the inference-job, but we'll start populating it and adding logic around it.
   -- We'll start this out as nullable, then migrate the old rows with a default column value.
+  --
+  -- Fal jobs have the type `fal_queue`.
   job_type VARCHAR(32) DEFAULT NULL,
 
   -- There is an index on this column.
@@ -103,6 +105,11 @@ CREATE TABLE generic_inference_jobs (
   -- And furthermore, not all download URLs have a cover image.
   maybe_cover_image_media_file_token VARCHAR(32) DEFAULT NULL,
 
+  -- If we pre-constructed a prompt, this will be copied to the resultant media generated.
+  -- This is especially important for Fal-related jobs and endpoints. We need this for keeping
+  -- track of all the prompt details, context images, etc.
+  maybe_prompt_token VARCHAR(30) DEFAULT NULL,
+
   -- A migration flag we can control from the backend API service which will tell
   -- the worker where to store the results. If false, GCP. If true, AWS.
   store_in_aws BOOLEAN NOT NULL DEFAULT FALSE,

_sql/migrations_squashed/media_files.sql

@@ -61,6 +61,9 @@ CREATE TABLE media_files (
   --   * 'workflow' (DEPRECATED)
   origin_product_category VARCHAR(16) NOT NULL DEFAULT "unknown",
 
+  -- What service provided the generation (if any).
+  maybe_generation_provider VARCHAR(16) DEFAULT NULL,
+
   -- For inference that can be tied back to a model, the type of model.
   -- There are multiple indices on this column!
   -- DO NOT EXPOSE THIS TO USERS VIA THE API, as we may leak secrets in doing so.

_sql/migrations_squashed/prompts/batch_generations.sql

@@ -15,9 +15,9 @@ CREATE TABLE batch_generations (
 
   -- The type of record this is attached to
   -- Currently supported values:
-  --   * 'model_weight'
+  --   * 'model_weight' (deprecated)
+  --   * 'comment' (deprecated)
   --   * 'media_file'
-  --   * 'comment'
   entity_type VARCHAR(32) NOT NULL,
 
   -- The token for the record.
@@ -37,6 +37,4 @@ CREATE TABLE batch_generations (
   -- NB: Realistically, we probably don't need this index until we have a lot of entity types, and the entity_token
   -- entropy is probably sufficient enough as-is.
   -- KEY index_entity_type_and_token (entity_type, entity_token),
-
-
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_sql/migrations_squashed/prompts/prompt_context_items.sql

@@ -0,0 +1,30 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- An n:m join table between prompts and media_files.
+-- This associates mostly image media files with prompts, though we could conceivably
+-- also store 3D sets, etc.
+CREATE TABLE prompt_context_items (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Foreign key, many-to-many
+  -- There is no uniqueness constraint for (prompt_token, media_token) because a user
+  -- might attach many duplicate images to the same prompt.
+  prompt_token VARCHAR(32) NOT NULL,
+
+  -- Foreign key to the media token
+  -- There is no uniqueness constraint for (prompt_token, media_token) because a user
+  -- might attach many duplicate images to the same prompt.
+  media_token VARCHAR(32) NOT NULL,
+
+  -- The semantic type of context.
+  context_semantic_type VARCHAR(16) NOT NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  KEY index_prompt_token (prompt_token),
+  KEY index_media_token (media_token)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_sql/migrations_squashed/prompts/prompts.sql

@@ -19,13 +19,25 @@ CREATE TABLE prompts (
   -- Original product to which the prompt applied.
   -- Prompts may or may not be compatible across products.
   -- Possible values:
-  --  * 'stable_diffusion'
-  --  * 'comfy_ui'
+  --  * 'artcraft_app'
+  --  * 'stable_diffusion' (legacy)
+  --  * 'comfy_ui' (legacy)
   prompt_type VARCHAR(16) NOT NULL,
 
-  -- Whomever created the prompt
+  -- What type of model was used to generate the result.
+  -- NB: We're conflating prompts with outputs, but in a
+  -- sense the model is inextricably part of the prompt.
+  maybe_model_type VARCHAR(32) DEFAULT NULL,
+
+  -- What service provided the generation.
+  maybe_generation_provider VARCHAR(16) NOT NULL,
+
+  -- Whoever created the prompt
   maybe_creator_user_token VARCHAR(32) DEFAULT NULL,
 
+  -- Whoever created the prompt (anonymous user tracking)
+  maybe_anonymous_visitor_token VARCHAR(32) DEFAULT NULL,
+
   -- ========== PROMPT DETAILS ==========
 
   -- The positive prompt (optional)

crates/api_clients/cloud_storage/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+name = "cloud_storage"
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+publish = false
+
+[lib]
+name = "cloud_storage"
+path = "src/lib.rs"
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["workspace-build-acceleration"]
+
+[dependencies]
+# Workspace hack (faster compile times)
+workspace-build-acceleration.workspace = true
+
+# Internal
+bucket_paths = { path = "../../lib/bucket_paths" }
+crockford.workspace = true
+easyenv = { path = "../../lib/easyenv" }
+errors.workspace = true
+filesys.workspace = true
+hashing = { path = "../../lib/files/hashing" }
+mimetypes.workspace = true
+storyteller_root = { path = "../../lib/storyteller_root" }
+
+# External
+anyhow.workspace = true
+log.workspace = true
+env_logger = "0.11.3"
+async-trait = "0.1.75"
+# The "never-encode-slash" feature seems to have been removed:
+# https://github.com/durch/rust-s3/pull/254
+rust-s3 = { version = "0.36.0-beta.2" }
+
+# Runtime
+tokio.workspace = true

crates/api_clients/fakeyou_client/Cargo.toml

@@ -8,27 +8,33 @@ authors = [
 ]
 publish = false
 
+[package.metadata.cargo-udeps.ignore]
+normal = ["workspace-build-acceleration"]
+
 [lib]
 name = "fakeyou_client"
 path = "src/lib.rs"
 
 [dependencies]
+# Workspace hack (faster compile times)
+workspace-build-acceleration.workspace = true
+
 # Internal
-enums = { path = "../../../schema/public/enums" }
-errors = { path = "../../errors" }
-tokens = { path = "../../../schema/public/tokens" }
+enums.workspace = true
+errors.workspace = true
+tokens.workspace = true
 
 # External
-bytes = "1.3"
-chrono = { version = "0.4.22", features = ["serde"] }
-log = "0.4.14"
-once_cell = "1.9.0"
+bytes.workspace = true
+chrono.workspace = true
+log.workspace = true
+once_cell.workspace = true
 
 # Serialization
-serde = "1.0.125"
-serde_derive = "1.0.125"
-serde_json = "1.0.64"
-toml = "0.5.8"
+serde.workspace = true
+serde_derive.workspace = true
+serde_json.workspace = true
+toml.workspace = true
 
 # HTTP
 backoff = { version = "0.4.0", features = ["futures"] } # Exponential backoff retry

crates/api_clients/fal_client/Cargo.toml

@@ -12,27 +12,27 @@ name = "fal_client"
 path = "src/lib.rs"
 
 [dependencies]
-
 # Workspace hack (faster compile times)
-workspace-build-acceleration = { workspace =  true }
+workspace-build-acceleration.workspace = true
 
 # Internal
-file_to_base64_url = { workspace = true }
-errors = { workspace = true }
+errors.workspace = true
+file_to_base64_url.workspace = true
 
 # External
-anyhow = { workspace = true }
-futures = { workspace = true }
-log = { workspace = true }
-reqwest = { workspace = true }
-serde = { workspace = true }
-tokio = { workspace = true }
-url = { workspace = true }
+anyhow.workspace = true
+futures.workspace = true
+log.workspace = true
+reqwest.workspace = true
+serde.workspace = true
+tokio.workspace = true
+url.workspace = true
 
 # For now, we'll vendor in fal.
 # We need to implement webhooks and the upstream doesn't support them yet.
 #fal = { version = "0.3.6", features = ["endpoints"] }
-fal = { path = "../../../vendor/fal-rs", features = ["endpoints"] }
+fal = { path = "../../vendor/fal-rs", features = ["endpoints"] }
 
 [dev-dependencies]
-testing = { path = "../../testing" }
+testing = { path = "../../lib/testing" }
+test_data.workspace = true

crates/api_clients/fal_client/src/creds/fal_api_key.rs

@@ -8,6 +8,7 @@ impl FalApiKey {
   }
   
   pub fn from_str(api_key: &str) -> Self {
-    Self(api_key.to_string())
+    let api_key = api_key.trim().to_string();
+    Self(api_key)
   }
 }

crates/api_clients/fal_client/src/creds/mod.rs

@@ -0,0 +1,2 @@
+pub mod fal_api_key;
+pub mod open_ai_api_key;

crates/api_clients/fal_client/src/creds/open_ai_api_key.rs

@@ -0,0 +1,15 @@
+
+/// This is for Fal's OpenAI "Bring Your Own Key" (BYOK) endpoints.
+#[derive(Clone)]
+pub struct OpenAiApiKey(pub String);
+
+impl OpenAiApiKey {
+  pub fn new(api_key: String) -> Self {
+    Self(api_key)
+  }
+
+  pub fn from_str(api_key: &str) -> Self {
+    let api_key = api_key.trim().to_string();
+    Self(api_key)
+  }
+}