copy paste bug fixes

Actual backend integration pending

.github/workflows/artcraft-macos-publish.yml

@@ -0,0 +1,232 @@
+name: "Artcraft MacOS Publish"
+
+on:
+  push:
+    branches:
+      - release
+      - artcraft-release
+      - artcraft-macos  # Additional branch for MacOS testing
+
+# This workflow creates Universal macOS builds with signing certificate support.
+# On each push to the release branches it will create or update a GitHub release, build your app, and upload the artifacts to the release.
+
+jobs:
+  publish-artcraft:
+    permissions:
+      contents: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Universal macOS builds (arm64 and x86_64)
+          - platform: 'macos-latest-xlarge'
+            args: '--target universal-apple-darwin'
+          # Uncomment below for additional platforms
+          # - platform: 'ubuntu-22.04'
+          #   args: ''
+          # - platform: 'windows-latest'
+          #   args: ''
+
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: lts/*
+
+      - name: install Rust stable
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.platform == 'macos-latest-xlarge' && 'aarch64-apple-darwin,x86_64-apple-darwin' || '' }}
+
+      - name: install dependencies (ubuntu only)
+        if: matrix.platform == 'ubuntu-22.04'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf
+
+      - name: install frontend dependencies
+        run: |
+          # Navigate to the frontend directory to install dependencies
+          cd ./frontend
+
+          npm cache clean --force
+
+          echo "Installing all workspace dependencies..."
+          npm ci
+
+          echo "=== Version Information ==="
+          echo "Node version:"
+          node --version
+          echo "NPM version:"
+          npm --version
+
+          # Remove mismatched @nx packages and reinstall with correct versions
+          npm install --save-dev @nx/remix@21.1.1 @nx/js@21.1.1 @nx/react@21.1.1 @nx/vite@21.1.1 @nx/eslint@21.1.1
+
+          echo "=== Nx versions after adjustment ==="
+          npm list --depth=0 | grep nx
+
+      - name: import Apple Developer Certificate
+        if: matrix.platform == 'macos-latest-xlarge'
+        env:
+          APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
+          APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
+          KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
+        run: |
+          echo "Starting certificate import process..."
+
+          # Create certificate file
+          echo $APPLE_CERTIFICATE | base64 --decode > certificate.p12
+
+          # Debug: Check if certificate file exists and has content
+          echo "Certificate file info:"
+          ls -la certificate.p12
+          file certificate.p12
+
+          # Debug: Check certificate contents without password (just structure)
+          echo "Certificate structure (may show error if password protected):"
+          openssl pkcs12 -info -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -noout || echo "Could not read certificate structure"
+
+          # Debug: Try to list what's in the p12 file
+          echo "Attempting to list certificate contents:"
+          security cms -D -i certificate.p12 || echo "Not a CMS file, trying PKCS12..."
+          openssl pkcs12 -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -info -nodes 2>&1 | head -20 || echo "Failed to read PKCS12 contents"
+
+          # Debug: Extract certificate details to verify type and expiration
+          echo "Extracting certificate details..."
+          # Extract just the certificate portion and examine it
+          openssl pkcs12 -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -nokeys 2>/dev/null | openssl x509 -text -noout | grep -E "(Subject:|Issuer:|Not Before|Not After)" || echo "Failed to parse certificate details"
+
+          # Also try to see what type of certificate this is
+          echo "Certificate type check:"
+          openssl pkcs12 -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -nokeys 2>/dev/null | openssl x509 -text -noout | grep -E "(CN=|OU=)" || echo "Could not determine certificate type"
+
+          # Debug: Check if private key exists
+          echo "Checking for private key..."
+          openssl pkcs12 -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -nocerts -nodes 2>&1 | grep -E "(PRIVATE KEY|RSA KEY)" | head -5 || echo "No private key found"
+
+          # Create temporary keychain
+          security create-keychain -p "$KEYCHAIN_PASSWORD" build.keychain
+          security default-keychain -s build.keychain
+          security unlock-keychain -p "$KEYCHAIN_PASSWORD" build.keychain
+          security set-keychain-settings -t 3600 -u build.keychain
+
+          # Import certificate - ensure both certificate and private key are imported
+          echo "Importing certificate into keychain..."
+          # Import the p12 directly - this is the most reliable method
+          security import certificate.p12 -k build.keychain -P "$APPLE_CERTIFICATE_PASSWORD" -T /usr/bin/codesign -T /usr/bin/security -T /usr/bin/productbuild
+
+          # Alternative: Try with additional flags if the first import didn't create an identity
+          echo "Verifying import..."
+          if ! security find-identity -v build.keychain | grep -q "1 valid identities found"; then
+            echo "First import didn't create identity, trying with -A flag..."
+            security import certificate.p12 -k build.keychain -P "$APPLE_CERTIFICATE_PASSWORD" -A
+          fi
+
+          # Set trust settings for the certificate
+          echo "Setting trust settings..."
+          security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$KEYCHAIN_PASSWORD" build.keychain
+
+          # Unlock the keychain again to ensure it's accessible
+          security unlock-keychain -p "$KEYCHAIN_PASSWORD" build.keychain
+
+          # Add the keychain to the search list
+          security list-keychains -d user -s build.keychain $(security list-keychains -d user | sed 's/^[[:space:]]*"//;s/"$//')
+
+          # List all certificates to debug
+          echo "Listing all keychains:"
+          security list-keychains
+
+          echo "Listing all certificates in build keychain:"
+          security find-certificate -a -p build.keychain || echo "No certificates found"
+
+          echo "Detailed certificate info:"
+          security find-certificate -c "Learning Machines" build.keychain || echo "Certificate 'Learning Machines' not found"
+
+          # Check certificate trust settings
+          echo "Checking certificate trust settings:"
+          security dump-trust-settings || echo "No custom trust settings"
+
+          # Try to find codesigning identities
+          echo "Finding all identities in keychain:"
+          security find-identity -v build.keychain || echo "No identities found"
+
+          echo "Finding codesigning identities:"
+          security find-identity -v -p codesigning build.keychain || echo "No codesigning identities found"
+
+          # Additional debug: Check if this is a valid Developer ID cert
+          echo "Checking for Developer ID certificates:"
+          security find-certificate -a build.keychain | grep -A 5 -B 5 "Developer ID" || echo "No Developer ID certificates found"
+
+      - name: verify certificate
+        if: matrix.platform == 'macos-latest-xlarge'
+        run: |
+          echo "=== Certificate Verification ==="
+
+          echo "1. Checking codesigning identities:"
+          security find-identity -v -p codesigning build.keychain || echo "No codesigning identities found"
+
+          echo -e "\n2. Checking all identities:"
+          security find-identity -v build.keychain || echo "No identities found"
+
+          echo -e "\n3. Checking certificates in keychain:"
+          security find-certificate -c "Learning Machines" build.keychain -p | openssl x509 -text -noout | grep -E "(Subject:|CN=)" || echo "Could not parse certificate"
+
+          # Set the correct signing identity
+          echo "APPLE_SIGNING_IDENTITY=Developer ID Application: Learning Machines LLC (DJ6XS33FX8)" >> $GITHUB_ENV
+          echo "Proceeding with identity: Developer ID Application: Learning Machines LLC (DJ6XS33FX8)"
+
+          # Also set the team ID
+          echo "APPLE_DEVELOPMENT_TEAM=DJ6XS33FX8" >> $GITHUB_ENV
+
+      - name: Pre-build frontend to avoid Tauri build issues
+        run: |
+          # Build the frontend app from the correct directory
+          cd ./frontend
+
+          # Increase Node.js memory limit for the build
+          export NODE_OPTIONS="--max-old-space-size=8192"
+
+          # Use npx to ensure we use the local nx version
+          echo "Building with npx nx..."
+          npx nx run editor3d:build
+
+          echo "Checking build output..."
+          ls -la apps/editor3d/dist/ | head -10
+
+          # Update Tauri config to skip the build (since we already built)
+          cd ${{ github.workspace }}/crates/desktop/artcraft
+          cp tauri.conf.json tauri.conf.json.bak
+          # Set beforeBuildCommand to empty or a no-op command
+          if command -v jq &> /dev/null; then
+            jq '.build.beforeBuildCommand = ""' tauri.conf.json > tauri.conf.json.tmp
+            mv tauri.conf.json.tmp tauri.conf.json
+          else
+            node -e "const fs = require('fs'); const config = JSON.parse(fs.readFileSync('tauri.conf.json', 'utf8')); config.build.beforeBuildCommand = ''; fs.writeFileSync('tauri.conf.json', JSON.stringify(config, null, 2));"
+          fi
+
+
+      - name: build and publish
+        uses: tauri-apps/tauri-action@v0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          APPLE_ID: ${{ secrets.APPLE_ID }}
+          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
+          APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
+          APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
+          APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
+          APPLE_SIGNING_IDENTITY: ${{ env.APPLE_SIGNING_IDENTITY }}
+          APPLE_DEVELOPMENT_TEAM: ${{ env.APPLE_DEVELOPMENT_TEAM }}
+          # Use the SQLx query cache for SQLite
+          SQLX_OFFLINE: true
+        with:
+          projectPath: ./crates/desktop/artcraft
+          tagName: artcraft-v__VERSION__ # the action automatically replaces __VERSION__ with the app version.
+          releaseName: "Artcraft v__VERSION__"
+          releaseBody: "See the assets to download this version and install."
+          releaseDraft: true
+          prerelease: false
+          args: ${{ matrix.args }}

.github/workflows/artcraft-windows-publish.yml

@@ -0,0 +1,96 @@
+name: "Artcraft Windows Publish"
+
+on:
+  push:
+    branches:
+      - release
+      - artcraft-release
+      - artcraft-windows  # Additional branch for Windows testing
+
+# This workflow creates Windows builds.
+# On each push to the release branches it will create or update a GitHub release, build your app, and upload the artifacts to the release.
+
+jobs:
+  publish-artcraft-windows:
+    permissions:
+      contents: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Windows builds
+          - platform: 'windows-latest'
+            args: ''
+
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: lts/*
+
+      - name: install Rust stable
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: install frontend dependencies
+        run: |
+          # Navigate to the frontend directory to install dependencies
+          cd ./frontend
+
+          npm cache clean --force
+
+          echo "Installing all workspace dependencies..."
+          npm ci
+
+          echo "=== Version Information ==="
+          echo "Node version:"
+          node --version
+          echo "NPM version:"
+          npm --version
+
+          # Remove mismatched @nx packages and reinstall with correct versions
+          npm install --save-dev @nx/remix@21.1.1 @nx/js@21.1.1 @nx/react@21.1.1 @nx/vite@21.1.1 @nx/eslint@21.1.1
+
+          echo "=== Nx versions after adjustment ==="
+          npm list --depth=0 | findstr nx
+
+      - name: Pre-build frontend to avoid Tauri build issues
+        run: |
+          # Build the frontend app from the correct directory
+          cd ./frontend
+
+          # Increase Node.js memory limit for the build
+          $env:NODE_OPTIONS="--max-old-space-size=8192"
+
+          # Use npx to ensure we use the local nx version
+          echo "Building with npx nx..."
+          npx nx run editor3d:build
+
+          echo "Checking build output..."
+          dir apps/editor3d/dist/ | Select-Object -First 10
+
+          # Update Tauri config to skip the build (since we already built)
+          cd ${{ github.workspace }}/crates/desktop/artcraft
+          Copy-Item tauri.conf.json tauri.conf.json.bak
+
+          # Set beforeBuildCommand to empty using PowerShell
+          $config = Get-Content tauri.conf.json | ConvertFrom-Json
+          $config.build.beforeBuildCommand = ""
+          $config | ConvertTo-Json -Depth 10 | Set-Content tauri.conf.json
+
+      - name: build and publish
+        uses: tauri-apps/tauri-action@v0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # Use the SQLx query cache for SQLite
+          SQLX_OFFLINE: true
+        with:
+          projectPath: ./crates/desktop/artcraft
+          tagName: artcraft-windows-v__VERSION__ # the action automatically replaces __VERSION__ with the app version.
+          releaseName: "Artcraft Windows v__VERSION__"
+          releaseBody: "Windows build - See the assets to download this version and install."
+          releaseDraft: true
+          prerelease: false
+          args: ${{ matrix.args }}

.gitignore

@@ -24,11 +24,13 @@ twitch_secrets.toml
 *-secrets.env
 
 # Utilities with secrets
-script/production_database_secret_backup.sh
-script/production_database_secret_connect.sh
 production_database_secret_connect.sh
 production_database_secret_query.sh
 production_database_voice_clone_report_download.sh
+script/artcraft/mac_signing_build.sh
+script/production_database_secret_backup.sh
+script/production_database_secret_connect.sh
+script/socat_dev_database_connect.sh
 
 # Scratch data
 scratch/

.sqlx/query-002cdb3d086f3b4181e2997befa54bd8516cb1d3d3bb6e389862695318d5c299.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\nINSERT INTO generic_inference_jobs\nSET\n  token = ?,\n  uuid_idempotency_token = ?,\n\n  job_type = ?,\n\n  maybe_external_third_party = ?,\n  maybe_external_third_party_id = ?,\n\n  product_category = ?,\n  inference_category = ?,\n\n  maybe_model_type = NULL,\n  maybe_model_token = NULL,\n\n  maybe_input_source_token = NULL,\n  maybe_input_source_token_type = NULL,\n\n  maybe_download_url = NULL,\n  maybe_cover_image_media_file_token = NULL,\n  \n  maybe_prompt_token = ?,\n\n  maybe_raw_inference_text = NULL,\n\n  maybe_inference_args = ?,\n\n  maybe_creator_user_token = ?,\n  maybe_creator_anonymous_visitor_token = ?,\n  creator_ip_address = ?,\n  creator_set_visibility = ?,\n\n  priority_level = 0,\n  is_keepalive_required = FALSE,\n  max_duration_seconds = 0,\n\n  is_debug_request = FALSE,\n  maybe_routing_tag = NULL,\n\n  status = ?\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 14
+    },
+    "nullable": []
+  },
+  "hash": "002cdb3d086f3b4181e2997befa54bd8516cb1d3d3bb6e389862695318d5c299"
+}

.sqlx/query-01a0bf33e8331c19dc73f064c54eaa863bec158ef7713450a06c75e246f5f08c.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "SQLite",
+  "query": "\n    INSERT INTO tasks (\n      id,\n      task_status,\n      task_type,\n      model_type,\n      provider,\n      provider_job_id,\n      frontend_caller,\n      frontend_subscriber_id,\n      frontend_subscriber_payload\n    )\n    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)\n  ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 9
+    },
+    "nullable": []
+  },
+  "hash": "01a0bf33e8331c19dc73f064c54eaa863bec158ef7713450a06c75e246f5f08c"
+}

.sqlx/query-08f52f15acf6afc7091a525e3dc88f706c71c3579e5e81aa9903807d85f469ef.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\nINSERT INTO prompts\nSET\n  token = ?,\n  prompt_type = ?,\n\n  maybe_creator_user_token = ?,\n  \n  maybe_model_type = ?,\n  maybe_generation_provider = ?,\n\n  maybe_positive_prompt = ?,\n  maybe_negative_prompt = ?,\n  \n  maybe_other_args = ?,\n\n  creator_ip_address = ?\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 9
+    },
+    "nullable": []
+  },
+  "hash": "08f52f15acf6afc7091a525e3dc88f706c71c3579e5e81aa9903807d85f469ef"
+}

.sqlx/query-094eb099cde85835db0662c2700c6b82644b51bf9b3609391ed4d2945fe5e98f.json

@@ -0,0 +1,69 @@
+{
+  "db_name": "MySQL",
+  "query": "\nSELECT\n    pci.media_token as `media_token: tokens::tokens::media_files::MediaFileToken`,\n    pci.context_semantic_type as `context_semantic_type: enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType`,\n\n    m.public_bucket_directory_hash,\n    m.maybe_public_bucket_prefix,\n    m.maybe_public_bucket_extension\n\nFROM prompt_context_items pci\nJOIN media_files m\nON\n    pci.media_token = m.token\nWHERE\n    pci.prompt_token = ?\nORDER BY pci.id ASC\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "media_token: tokens::tokens::media_files::MediaFileToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 1,
+        "name": "context_semantic_type: enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 64
+        }
+      },
+      {
+        "ordinal": 2,
+        "name": "public_bucket_directory_hash",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 3,
+        "name": "maybe_public_bucket_prefix",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 64
+        }
+      },
+      {
+        "ordinal": 4,
+        "name": "maybe_public_bucket_extension",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 64
+        }
+      }
+    ],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      true,
+      true
+    ]
+  },
+  "hash": "094eb099cde85835db0662c2700c6b82644b51bf9b3609391ed4d2945fe5e98f"
+}

.sqlx/query-13ca06bcbb4a576f8a2229a8554b7488d379baa381d8d5cb60972750e81b2173.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "MySQL",
-  "query": "\nINSERT INTO prompts\nSET\n  token = ?,\n  prompt_type = ?,\n\n  maybe_creator_user_token = ?,\n\n  maybe_positive_prompt = ?,\n  maybe_negative_prompt = ?,\n  maybe_other_args = ?,\n\n  creator_ip_address = ?\n        ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 7
-    },
-    "nullable": []
-  },
-  "hash": "13ca06bcbb4a576f8a2229a8554b7488d379baa381d8d5cb60972750e81b2173"
-}

.sqlx/query-19708db99f2b3c6283d2115a511e0eb1c99fab396ad0effd9eb2dd79926e9c93.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\nUPDATE generic_inference_jobs\nSET\n  status = 'cancelled_by_system',\n  retry_at = NULL,\n  internal_debugging_failure_reason = 'reap_stale_fakeyou_jobs'\nWHERE\n  job_type != 'fal_queue'\n  AND status IN (\n    'pending',\n    'started',\n    'attempt_failed'\n   )\n   AND created_at > NOW() - INTERVAL 30 MINUTE\nLIMIT 5000\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 0
+    },
+    "nullable": []
+  },
+  "hash": "19708db99f2b3c6283d2115a511e0eb1c99fab396ad0effd9eb2dd79926e9c93"
+}

.sqlx/query-61744bbf83f7cf00348bcc21e868d9c506fec46da4a33ac50d63c6d9e00c0cec.json

@@ -1,102 +0,0 @@
-{
-  "db_name": "MySQL",
-  "query": "\nSELECT\n    p.token as `token: tokens::tokens::prompts::PromptToken`,\n\n    p.prompt_type as `prompt_type: enums::by_table::prompts::prompt_type::PromptType`,\n\n    p.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n\n    p.maybe_positive_prompt,\n    p.maybe_negative_prompt,\n    p.maybe_other_args,\n\n    p.creator_ip_address,\n    p.created_at\n\nFROM prompts as p\nWHERE\n    p.token = ?\n        ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "token: tokens::tokens::prompts::PromptToken",
-        "type_info": {
-          "type": "VarString",
-          "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
-          "char_set": 224,
-          "max_size": 128
-        }
-      },
-      {
-        "ordinal": 1,
-        "name": "prompt_type: enums::by_table::prompts::prompt_type::PromptType",
-        "type_info": {
-          "type": "VarString",
-          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
-          "char_set": 224,
-          "max_size": 64
-        }
-      },
-      {
-        "ordinal": 2,
-        "name": "maybe_creator_user_token: tokens::tokens::users::UserToken",
-        "type_info": {
-          "type": "VarString",
-          "flags": "MULTIPLE_KEY | BINARY",
-          "char_set": 224,
-          "max_size": 128
-        }
-      },
-      {
-        "ordinal": 3,
-        "name": "maybe_positive_prompt",
-        "type_info": {
-          "type": "Blob",
-          "flags": "BLOB | BINARY",
-          "char_set": 224,
-          "max_size": 262140
-        }
-      },
-      {
-        "ordinal": 4,
-        "name": "maybe_negative_prompt",
-        "type_info": {
-          "type": "Blob",
-          "flags": "BLOB | BINARY",
-          "char_set": 224,
-          "max_size": 262140
-        }
-      },
-      {
-        "ordinal": 5,
-        "name": "maybe_other_args",
-        "type_info": {
-          "type": "Blob",
-          "flags": "BLOB | BINARY",
-          "char_set": 224,
-          "max_size": 262140
-        }
-      },
-      {
-        "ordinal": 6,
-        "name": "creator_ip_address",
-        "type_info": {
-          "type": "VarString",
-          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
-          "char_set": 224,
-          "max_size": 160
-        }
-      },
-      {
-        "ordinal": 7,
-        "name": "created_at",
-        "type_info": {
-          "type": "Timestamp",
-          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP",
-          "char_set": 63,
-          "max_size": 19
-        }
-      }
-    ],
-    "parameters": {
-      "Right": 1
-    },
-    "nullable": [
-      false,
-      false,
-      true,
-      true,
-      true,
-      true,
-      false,
-      false
-    ]
-  },
-  "hash": "61744bbf83f7cf00348bcc21e868d9c506fec46da4a33ac50d63c6d9e00c0cec"
-}

.sqlx/query-90366e1705a0343bdfe4fc472d701b8e1dbd3a2dabedaadb633ac3542de72fa7.json

@@ -48,7 +48,7 @@
         "name": "maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken",
         "type_info": {
           "type": "VarString",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
           "char_set": 224,
           "max_size": 128
         }

.sqlx/query-b4470e607c005421dd8504b9f1d695080a581936ac6ce00b6f437b2764e741be.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\n        INSERT INTO media_files\n        SET\n            token = ?,\n\n            media_class = ?,\n            media_type = ?,\n\n            is_user_upload = ?,\n            is_intermediate_system_file = ?,\n\n            origin_category = ?, \n            origin_product_category = ?, \n            maybe_origin_model_type = ?, \n            maybe_origin_model_token = ?, \n            maybe_origin_filename = ?,\n\n            is_batch_generated = ?,\n            maybe_batch_token = ?,\n\n            maybe_mime_type = ?,\n            file_size_bytes = ?,\n            maybe_duration_millis = ?,\n            maybe_audio_encoding = ?, \n            maybe_video_encoding = ?, \n            maybe_frame_width = ?, \n            maybe_frame_height = ?,\n            maybe_prompt_token = ?,\n            checksum_sha2 = ?,\n            \n            maybe_engine_category = ?,\n\n            maybe_title = ?,\n            maybe_text_transcript = ?,\n\n            maybe_scene_source_media_file_token = ?,\n\n            public_bucket_directory_hash = ?, \n            maybe_public_bucket_prefix = ?, \n            maybe_public_bucket_extension = ?, \n\n            maybe_creator_user_token = ?, \n            maybe_creator_anonymous_visitor_token = ?, \n\n            creator_ip_address = ?, \n            creator_set_visibility = ?, \n\n            maybe_creator_file_synthetic_id = ?, \n            maybe_creator_category_synthetic_id = ?,\n\n            extra_file_modification_info = ?,\n\n            maybe_mod_user_token = ?, \n            is_generated_on_prem = ?, \n            generated_by_worker = ?, \n            generated_by_cluster = ?\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 39
+    },
+    "nullable": []
+  },
+  "hash": "b4470e607c005421dd8504b9f1d695080a581936ac6ce00b6f437b2764e741be"
+}

.sqlx/query-b46b0d022790f4e10f6e47126df0fdc55172bf8cda77d4e0b2afac9e6ebd1630.json

@@ -0,0 +1,124 @@
+{
+  "db_name": "MySQL",
+  "query": "\nSELECT\n    p.token as `token: tokens::tokens::prompts::PromptToken`,\n\n    p.prompt_type as `prompt_type: enums::by_table::prompts::prompt_type::PromptType`,\n\n    p.maybe_model_type as `maybe_model_type: enums::common::model_type::ModelType`,\n    p.maybe_generation_provider as `maybe_generation_provider: enums::common::generation_provider::GenerationProvider`,\n\n    p.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n\n    p.maybe_positive_prompt,\n    p.maybe_negative_prompt,\n    p.maybe_other_args,\n\n    p.creator_ip_address,\n    p.created_at\n\nFROM prompts as p\nWHERE\n    p.token = ?\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "token: tokens::tokens::prompts::PromptToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 1,
+        "name": "prompt_type: enums::by_table::prompts::prompt_type::PromptType",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 64
+        }
+      },
+      {
+        "ordinal": 2,
+        "name": "maybe_model_type: enums::common::model_type::ModelType",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 3,
+        "name": "maybe_generation_provider: enums::common::generation_provider::GenerationProvider",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 64
+        }
+      },
+      {
+        "ordinal": 4,
+        "name": "maybe_creator_user_token: tokens::tokens::users::UserToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "MULTIPLE_KEY | BINARY",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 5,
+        "name": "maybe_positive_prompt",
+        "type_info": {
+          "type": "Blob",
+          "flags": "BLOB | BINARY",
+          "char_set": 224,
+          "max_size": 262140
+        }
+      },
+      {
+        "ordinal": 6,
+        "name": "maybe_negative_prompt",
+        "type_info": {
+          "type": "Blob",
+          "flags": "BLOB | BINARY",
+          "char_set": 224,
+          "max_size": 262140
+        }
+      },
+      {
+        "ordinal": 7,
+        "name": "maybe_other_args",
+        "type_info": {
+          "type": "Blob",
+          "flags": "BLOB | BINARY",
+          "char_set": 224,
+          "max_size": 262140
+        }
+      },
+      {
+        "ordinal": 8,
+        "name": "creator_ip_address",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 160
+        }
+      },
+      {
+        "ordinal": 9,
+        "name": "created_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP",
+          "char_set": 63,
+          "max_size": 19
+        }
+      }
+    ],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": [
+      false,
+      false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      false,
+      false
+    ]
+  },
+  "hash": "b46b0d022790f4e10f6e47126df0fdc55172bf8cda77d4e0b2afac9e6ebd1630"
+}

.sqlx/query-c58f414b5e4d0f0420d1894e2f40311e5613919f49f3ac91f81f5fc2bef1bea8.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "MySQL",
-  "query": "\n        INSERT INTO media_files\n        SET\n            token = ?,\n\n            media_class = ?,\n            media_type = ?,\n\n            is_user_upload = ?,\n            is_intermediate_system_file = ?,\n\n            origin_category = ?, \n            origin_product_category = ?, \n            maybe_origin_model_type = ?, \n            maybe_origin_model_token = ?, \n            maybe_origin_filename = ?,\n\n            is_batch_generated = ?,\n            maybe_batch_token = ?,\n\n            maybe_mime_type = ?,\n            file_size_bytes = ?,\n            maybe_duration_millis = ?,\n            maybe_audio_encoding = ?, \n            maybe_video_encoding = ?, \n            maybe_frame_width = ?, \n            maybe_frame_height = ?,\n            maybe_prompt_token = ?,\n            checksum_sha2 = ?,\n\n            maybe_title = ?,\n            maybe_text_transcript = ?,\n\n            maybe_scene_source_media_file_token = ?,\n\n            public_bucket_directory_hash = ?, \n            maybe_public_bucket_prefix = ?, \n            maybe_public_bucket_extension = ?, \n\n            maybe_creator_user_token = ?, \n            maybe_creator_anonymous_visitor_token = ?, \n\n            creator_ip_address = ?, \n            creator_set_visibility = ?, \n\n            maybe_creator_file_synthetic_id = ?, \n            maybe_creator_category_synthetic_id = ?,\n\n            extra_file_modification_info = ?,\n\n            maybe_mod_user_token = ?, \n            is_generated_on_prem = ?, \n            generated_by_worker = ?, \n            generated_by_cluster = ?\n        ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 38
-    },
-    "nullable": []
-  },
-  "hash": "c58f414b5e4d0f0420d1894e2f40311e5613919f49f3ac91f81f5fc2bef1bea8"
-}

.sqlx/query-dc0367e9e43c3d7dda3623e77337ab4529eac2192475be7d08589886703ba1cb.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "SQLite",
+  "query": "\n    UPDATE tasks\n    SET task_status = ?\n    WHERE id = ?\n  ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 2
+    },
+    "nullable": []
+  },
+  "hash": "dc0367e9e43c3d7dda3623e77337ab4529eac2192475be7d08589886703ba1cb"
+}

.sqlx/query-f68a8917e639418a0bb952bef606c96299001e143c745ee30df99b72d1235d35.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\nUPDATE generic_inference_jobs\nSET\n  status = ?,\n  on_success_result_entity_type = ?,\n  on_success_result_entity_token = ?,\n  failure_reason = NULL,\n  internal_debugging_failure_reason = NULL,\n  success_execution_millis = NULL,\n  success_inference_execution_millis = NULL,\n  retry_at = NULL,\n  successfully_completed_at = NOW()\nWHERE token = ?\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 4
+    },
+    "nullable": []
+  },
+  "hash": "f68a8917e639418a0bb952bef606c96299001e143c745ee30df99b72d1235d35"
+}

.sqlx/query-fbe482c8987231c3b5d75f2e792d0fdd033de8a7eb821428fa034752c73edfb7.json

@@ -0,0 +1,124 @@
+{
+  "db_name": "MySQL",
+  "query": "\nSELECT\n    jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,\n\n    jobs.status as `status: enums::common::job_status_plus::JobStatusPlus`,\n    \n    jobs.maybe_external_third_party as `external_third_party: enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty`,\n    jobs.maybe_external_third_party_id as `external_third_party_id`,\n\n    jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n    jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,\n    jobs.creator_ip_address,\n    \n    jobs.maybe_prompt_token as `maybe_prompt_token: tokens::tokens::prompts::PromptToken`,\n\n    jobs.created_at,\n    jobs.updated_at\n\nFROM generic_inference_jobs as jobs\n\nWHERE jobs.maybe_external_third_party = ?\nAND jobs.maybe_external_third_party_id = ?\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 1,
+        "name": "status: enums::common::job_status_plus::JobStatusPlus",
+        "type_info": {
+          "type": "String",
+          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | ENUM",
+          "char_set": 224,
+          "max_size": 76
+        }
+      },
+      {
+        "ordinal": 2,
+        "name": "external_third_party: enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty",
+        "type_info": {
+          "type": "VarString",
+          "flags": "MULTIPLE_KEY | BINARY",
+          "char_set": 224,
+          "max_size": 64
+        }
+      },
+      {
+        "ordinal": 3,
+        "name": "external_third_party_id",
+        "type_info": {
+          "type": "VarString",
+          "flags": "MULTIPLE_KEY | BINARY",
+          "char_set": 224,
+          "max_size": 256
+        }
+      },
+      {
+        "ordinal": 4,
+        "name": "maybe_creator_user_token: tokens::tokens::users::UserToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "MULTIPLE_KEY | BINARY",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 5,
+        "name": "maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "MULTIPLE_KEY | BINARY",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 6,
+        "name": "creator_ip_address",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 160
+        }
+      },
+      {
+        "ordinal": 7,
+        "name": "maybe_prompt_token: tokens::tokens::prompts::PromptToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 8,
+        "name": "created_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP",
+          "char_set": 63,
+          "max_size": 19
+        }
+      },
+      {
+        "ordinal": 9,
+        "name": "updated_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
+          "char_set": 63,
+          "max_size": 19
+        }
+      }
+    ],
+    "parameters": {
+      "Right": 2
+    },
+    "nullable": [
+      false,
+      false,
+      true,
+      true,
+      true,
+      true,
+      false,
+      true,
+      false,
+      false
+    ]
+  },
+  "hash": "fbe482c8987231c3b5d75f2e792d0fdd033de8a7eb821428fa034752c73edfb7"
+}

.vscode/settings.json

@@ -1,6 +1,15 @@
 {
-    "files.exclude": {
-        "**/*~": true,
-        "frontend_old/**": true
-    }
+  "files.exclude": {
+    "**/*~": true,
+    ".sqlx/**": true,
+    "_elasticsearch/**": true,
+    "_sql/**": true,
+    "frontend_old/**": true,
+    "includes/**": true,
+    "target/**": true,
+    "test_data/**": true,
+    "tmp/**": true,
+    "virtualenv/**": true,
+    "windows_dlls/**": true,
+  }
 }

Cargo.toml

@@ -4,12 +4,17 @@ resolver = "2"
 # This block defines the workspace members.
 # When we add a library or binary crate, it needs an entry here.
 members = [
-  # Types: common strongly-typed tokens, ids, payloads, etc.
-  "crates/_types/composite_identifiers",
-  "crates/_types/enums",
-  "crates/_types/enums_public",
-  "crates/_types/reusable_types", # TODO(bt, 2022-12-21): Deprecate
-  "crates/_types/tokens",
+  # API clients for external services
+  # These might be custom implementations or wrappers over other client libraries.
+  "crates/api_clients/cloud_storage",
+  "crates/api_clients/fal_client",
+  "crates/api_clients/google_drive_common",
+  "crates/api_clients/google_sign_in",
+  "crates/api_clients/openai_sora_client",
+  "crates/api_clients/openart_client",
+  "crates/api_clients/runwayml_client",
+  "crates/api_clients/storyteller_client",
+  "crates/api_clients/thumbnail_generator",
 
   # CLI tools for operations, etc.
   "crates/cli/devenv/dev-database-seed",
@@ -29,30 +34,24 @@ members = [
   # Shared libraries
   "crates/lib/actix_cors_configs",
   "crates/lib/actix_helpers",
-  "crates/lib/api_clients/cloud_storage",
-  "crates/lib/api_clients/fal_client",
-  "crates/lib/api_clients/google_drive_common",
-  "crates/lib/api_clients/google_sign_in",
-  "crates/lib/api_clients/openai_sora_client",
-  "crates/lib/api_clients/runwayml_client",
-  "crates/lib/api_clients/storyteller_client",
-  "crates/lib/api_clients/thumbnail_generator",
   "crates/lib/bootstrap",
   "crates/lib/bucket_paths",
+  "crates/lib/build_metadata",
   "crates/lib/caching/memory_caching",
   "crates/lib/caching/memory_store",
   "crates/lib/caching/redis_caching",
+  "crates/lib/cloudflare_errors",
   "crates/lib/concurrency",
   "crates/lib/cookies",
   "crates/lib/data/collections",
   "crates/lib/data/crockford",
   "crates/lib/data/datetimes",
   "crates/lib/data/primitives",
+  "crates/lib/data/web_base64",
   "crates/lib/deprecated/config", # TODO(bt, 2023-02-13): Deprecate or repurpose
   "crates/lib/deprecated/http_server_common", # TODO(bt, 2023-03-20): Move useful pieces to actix_helpers
   "crates/lib/dimensions",
   "crates/lib/easyenv", # NB: Vendored version of https://github.com/echelon/easyenv.rs that we'll improve.
-  "crates/lib/elasticsearch_schema",
   "crates/lib/email_sender",
   "crates/lib/errors",
   "crates/lib/files/archives",
@@ -68,27 +67,40 @@ members = [
   "crates/lib/jobs_common",
   "crates/lib/jwt_light",
   "crates/lib/logging",
-  "crates/lib/machine_learning/ml_models",
   "crates/lib/machine_learning/ml_weights_registry",
   "crates/lib/machine_learning/prompt_engineering",
   "crates/lib/markdown",
   "crates/lib/migration",
-  "crates/lib/mysql_queries",
   "crates/lib/password",
-  "crates/lib/redis_common",
-  "crates/lib/redis_schema",
   "crates/lib/server_environment",
   "crates/lib/shared_service_components",
   "crates/lib/sqlx_mysql_helpers",
   "crates/lib/storyteller_root",
   "crates/lib/subprocess_common",
+  "crates/lib/test_data",
   "crates/lib/testing",
   "crates/lib/tts_common",
   "crates/lib/url_config",
   "crates/lib/user_input_common",
   "crates/lib/web_scrapers",
   #"crates/lib/api_clients/fakeyou_client",
-  #"crates/lib/sqlite_queries",
+  #"crates/lib/machine_learning/ml_models",
+
+  # Schema (database): queries, database layouts, Redis keys, etc.
+  "crates/schema/database/elasticsearch_schema",
+  "crates/schema/database/mysql_queries",
+  "crates/schema/database/redis_common",
+  "crates/schema/database/redis_schema",
+  "crates/schema/database/sqlite_tasks",
+  #"crates/schema/database/sqlite_queries",
+
+  # Schema (public): common strongly-typed tokens, ids, payloads, etc.
+  "crates/schema/public/api/artcraft_api_defs",
+  "crates/schema/public/composite_identifiers",
+  "crates/schema/public/enums",
+  "crates/schema/public/enums_public",
+  "crates/schema/public/reusable_types", # TODO(bt, 2022-12-21): Deprecate
+  "crates/schema/public/tokens",
 
   # Services: API servers, jobs, workers, etc.
   "crates/service/job/analytics_job",
@@ -109,19 +121,23 @@ members = [
   "crates/service/plugins/user_traits_component",
 
   # Vendored code
-  "crates/vendor/candle/candle-core",
-  "crates/vendor/candle/candle-datasets",
-  "crates/vendor/candle/candle-examples",
-  "crates/vendor/candle/candle-flash-attn",
-  "crates/vendor/candle/candle-kernels",
-  "crates/vendor/candle/candle-metal-kernels",
-  "crates/vendor/candle/candle-nn",
-  "crates/vendor/candle/candle-onnx",
-  "crates/vendor/candle/candle-pyo3",
-  "crates/vendor/candle/candle-transformers",
-  "crates/vendor/candle/tensor-tools",
+  "crates/vendor/fal-rs",
+  "crates/vendor/fal-rs/fal-derive",
+  "crates/vendor/fal-rs/generate",
   "crates/vendor/limitation",
   "crates/vendor/tauri-plugin-http",
+  # NB(bt,2025-05-25): Too many failures on Mac. Disable for now.
+  #"crates/vendor/candle/candle-core",
+  #"crates/vendor/candle/candle-datasets",
+  #"crates/vendor/candle/candle-examples",
+  #"crates/vendor/candle/candle-flash-attn",
+  #"crates/vendor/candle/candle-kernels",
+  #"crates/vendor/candle/candle-metal-kernels",
+  #"crates/vendor/candle/candle-nn",
+  #"crates/vendor/candle/candle-onnx",
+  #"crates/vendor/candle/candle-pyo3",
+  #"crates/vendor/candle/candle-transformers",
+  #"crates/vendor/candle/tensor-tools",
   # NB(bt,2023-12-12): Newrelic keeps breaking with upgrades. Kill for now.
   #"crates/vendor/newrelic-telemetry-sdk-rust",
 
@@ -161,8 +177,19 @@ workspace-build-acceleration = { path = "crates/workspace-build-acceleration" }
 # External libs
 # Centrally pinned versions so that we can uniformly upgrade/downgrade versions across the workspace
 # If a crate needs to use an older/newer version, the dependency should be managed in that crate's own Cargo.toml
+actix = "0.13.5"
+actix-cors = "0.6.4" # TODO(bt): Carefully upgrade this. CORS broke last time.
+actix-http = "3.7.0"
+actix-multipart = "0.6.2"
+actix-rt = "2.10.0"
+actix-service = "2.0.3"
+actix-utils = "3.0.1"
+actix-web = "4.7.0" # TODO(bt): We're behind and need to bump versions.
+actix-web-actors = "4.3.1+deprecated"
+actix-web-lab = "0.20.2"
 anyhow = "1.0.98"
 base64 = "0.22.1"
+bytes = "1.10.1"
 chrono = { version = "0.4.41", features = ["serde"] }
 cookie = "0.18.1"
 futures = { version = "0.3.31" }
@@ -176,20 +203,27 @@ reqwest = { version = "0.12.15", features = ["json", "cookies"] }
 serde = {  version = "1.0.219", features = ["derive"] }
 serde_derive = "1.0.219"
 serde_json = "1.0.140"
-serde_yaml = "0.9.1"
+serde_yaml = "0.9.34+deprecated"
 sha3 = "0.10.8"
 tauri = { version = "2.4.1", features = ["unstable", "devtools"] } # NB: "devtools" prevents App Store distribution
+tempdir = "0.3.7"
+tempfile = "3.20.0"
 thiserror = "2.0.12"
-tokio = { version = "1.45.0", features = ["macros"] }
+tokio = { version = "1.45.1", features = ["macros"] }
 toml = "0.8.22"
 url = { version = "2.5.4", features = ["serde"] }
 
 # Internal libs
+artcraft_api_defs = { path = "crates/schema/public/api/artcraft_api_defs" }
+build_metadata = { path = "crates/lib/build_metadata" }
+cloud_storage = { path = "crates/api_clients/cloud_storage" }
+cloudflare_errors = { path = "crates/lib/cloudflare_errors" }
 crockford = { path = "crates/lib/data/crockford" }
-enums = { path = "crates/_types/enums" }
-enums_public = { path = "crates/_types/enums_public" }
+enums = { path = "crates/schema/public/enums" }
+enums_public = { path = "crates/schema/public/enums_public" }
 errors = { path = "crates/lib/errors" }
-fal_client = { path = "crates/lib/api_clients/fal_client" }
+fal_client = { path = "crates/api_clients/fal_client" }
+file_to_base64_url = { path = "crates/lib/files/file_to_base64_url" }
 filesys = { path = "crates/lib/files/filesys" }
 idempotency = { path = "crates/lib/idempotency" }
 images = { path = "crates/lib/files/images" }
@@ -197,9 +231,12 @@ jwt_light = { path = "crates/lib/jwt_light" }
 memory_caching = { path = "crates/lib/caching/memory_caching" }
 memory_store = { path = "crates/lib/caching/memory_store" }
 mimetypes = { path = "crates/lib/files/mimetypes" }
-openai_sora_client = { path = "crates/lib/api_clients/openai_sora_client" }
+openai_sora_client = { path = "crates/api_clients/openai_sora_client" }
 primitives = { path = "crates/lib/data/primitives" }
 shared_service_components = { path = "crates/lib/shared_service_components" }
-storyteller_client = { path = "crates/lib/api_clients/storyteller_client" }
+sqlite_tasks = { path = "crates/schema/database/sqlite_tasks" }
+storyteller_client = { path = "crates/api_clients/storyteller_client" }
 subprocess_common = { path = "crates/lib/subprocess_common" }
-tokens = { path = "crates/_types/tokens" }
+test_data = { path = "crates/lib/test_data" }
+tokens = { path = "crates/schema/public/tokens" }
+web_base64 = { path = "crates/lib/data/web_base64" }

README.md

@@ -1,7 +1,7 @@
 storyteller-rust
 ================
 
-Our Rust monorepo, which serves both our API surface area and our downloadable client applications.
+Our Artcraft monorepo, which serves our desktop application, backend infrastructure, and more.
 
 ```
                  ."-,.__
@@ -45,5 +45,6 @@ Our Rust monorepo, which serves both our API surface area and our downloadable c
                  / ,"'"\,'               `/  `-.|" 
 ```
 
-- [documentation](./_develop/docs)
-- [tools, misc](./_develop)
+- [developer documentation](./_docs)
+- [server API reference](https://storyteller-docs.netlify.app/)
+- [tools, scripts, misc](./script)

_develop/docs/local_development/general_setup.md

@@ -1,395 +0,0 @@
-FakeYou Development Setup (bare metal)
-======================================
-
-This document describes how to run the FakeYou infrastructure on your development 
-machine's bare metal (no containers).
-
-Setup instructions and development were tested and known to work on the following
-platforms:
-
-- Mac OS (Apple M2 silicon)
-- Ubuntu 22.04
-- Windows Subsystem for Linux (Ubuntu 22.04 distro) aka "WSL"
-
-The applications (binary targets) you might be interested in running include, but 
-are not limited to:
-
-- `storyteller-web`, the HTTP API server
-- `download-job`, which downloads models from the web and stores them in GCS
-- `inference-job`, which downloads models from GCS and runs inference with them
-
-Setup Instructions
-------------------
-
-### (1) Install Rust
-
-Install Rust [using this guide](https://www.rust-lang.org/learn/get-started). If it asks, 
-you'll want "stable" Rust, not "nightly" Rust. If it doesn't ask, it defaults to "stable".
-
-For Windows Subsystem for Linux, you'll want the WSL version, not the `.exe` file!
-
-### (2) Install the necessary libraries
-
-#### Mac
-
-On an up-to-date M1/M2 Mac, there should be no libraries you need to install! Any exceptions 
-to this assumption should be documented here.
-
-#### Ubuntu 22.04 and WSL:
-
-If you're on Ubuntu or WSL, you'll need to install some libraries to build the Rust code: 
-
-```bash
-# If you haven't run apt before, you'll need to fetch the package list:
-sudo apt update
-
-# Install the dependencies
-sudo apt install jq \
-  libmysqlclient-dev \
-  libsqlite3-dev \
-  pkgconf
-```
-
-(Note: jq is for combining mysql codegen outputs; pkgconf is if using openssl instead of rustls.)
-
-### (3) Install MySQL server
-
-#### Mac
-
-```bash
-# Install MySQL
-brew install mysql
-
-# Try to connect
-mysql -uroot
-```
-
-In the event of failure to connect, try the following steps:
-
-```bash
-# If connection fails, start service
-brew services start mysql
-
-# Try to connect again 
-# If this fails, reboot the machine, eg. using `sudo reboot now`
-mysql -uroot
-```
-
-#### Ubuntu 22.04 and WSL
-
-```bash
-# Install MySQL
-sudo apt install mysql-server
-
-# Try to connect 
-# The default password is typically "root".
-sudo mysql -u root -p
-````
-
-In the event of failure to connect, try the following steps:
-
-```bash
-# If it failed, you may need to start MySQL (typically only on WSL, not on native Ubuntu)
-sudo service mysql start
-
-# You may also have to change some socket file permissions (again, typically only on WSL)
-sudo chmod g+rx /var/run/mysqld
-sudo usermod -aG mysql $USER
-newgrp mysql
-
-# Try to connect again
-# The default password is typically "root".
-# If this fails, try rebooting. If it still fails, read some of the suggestions at the bottom 
-# of this README.
-sudo mysql -u root -p
-```
-
-### (4) Install a `storyteller` user and table in MySQL
-
-First, connect to MySQL:
-
-```bash
-# If the following command asks for a password, the password is typically "root"
-sudo mysql -u root -p
-```
-
-Once in MySQL, run the following to set up the database and user accounts:
-
-```mysql
-use mysql;
-CREATE DATABASE storyteller;
-CREATE USER 'storyteller'@'localhost' IDENTIFIED BY 'password';
-GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';
-```
-
-Then verify access with `./dev_mysql_connect.sh`
-
-### (5) Install Diesel CLI (for MySQL migrations):
-
-In order to run MySQL database migrations, you'll need to install the migration tool we use:
-
-```bash
-cargo install diesel_cli \
-  --no-default-features \
-  --features mysql,sqlite
-```
-
-In older Macs, there [have been some issues with Diesel CLI](https://github.com/diesel-rs/diesel/issues/2605)
-that require a few extra dependencies to be installed:
-
-```bash
-# If you're on Mac and the above command didn't work, run the following and then retry:
-brew install libpq
-```
-
-### (6) Run the pending database migrations:
-
-You're ready to run the migrations:
-
-```bash
-diesel migration run
-```
-
-You might get a scary message about `"Encountered unknown type for Mysql: enum"` -- you can safely ignore this 
-error if you see it in isolation. It doesn't impact the migrations whatsoever. (See notes at the bottom of 
-the README for details.)
-
-### (7) (optional) Install the SQLx CLI (if doing database development)
-
-(You can probably skip this step unless you're a backend engineer.) 
-
-If (and only if) you're doing Rust service development that changes the MySQL queries, then you should install 
-SQLx utilities. This tool is used to cache the column types and query plans for CI, as SQLx statically verifies 
-raw MySQL queries against the schema and Rust types.
-
-We'll be using diesel to manage the migrations, but sqlx within the app to actually perform queries.
-Diesel is an immature ORM, which isn't a good tech bet, so we use sqlx as at-compile-time
-typesafe SQL. 
-
-(TODO: Reevaluate this assessment for 2023-2024 and beyond. Maybe we do want Diesel now.)
-
-```bash
-# NB(bt,2023-10-04): "--no-default-features" not working
-# cargo install sqlx-cli --no-default-features --features rustls,mysql,sqlite
-cargo install sqlx-cli --features rustls,mysql,sqlite
-```
-
-### (8) Install hosts file:
-
-(This is the same for Linux, Mac, and WSL.)
-
-If you're developing against the frontend, it'll target development domains (eg. `dev.fakeyou.com`) instead 
-of `127.0.0.1` or `localhost`. You can make your machine route domains to localhost by editing your hosts 
-file (located at `/etc/hosts`) to include the following configuration lines:
-
-```
-# 1) Edit hosts file with your editor of choice
-#
-#  - sudo vi /etc/hosts
-#  - sudo nano /etc/hosts
-#
-# 2) Then paste:
-#
-
-127.0.0.1    dev.fakeyou.com
-127.0.0.1    api.dev.fakeyou.com
-127.0.0.1    devproxy.fakeyou.com
-
-127.0.0.1    dev.storyteller.ai
-127.0.0.1    api.dev.storyteller.ai
-127.0.0.1    devproxy.storyteller.ai
-```
-
-### (9) Install Redis
-
-#### Mac:
-
-```bash
-# Install Redis
-brew install redis
-
-# Start the service:
-brew services start redis
-```
-
-#### Ubuntu 22.04 and WSL
-
-```
-# Install Redis
-sudo apt install redis
-
-# Start the service (typically only needed on WSL):
-sudo service redis-server start
-```
-
-### (10) Install Elasticsearch (optional)
-
-(You can most likely skip this step.)
-
-Mac: (TODO - haven't installed yet)
-
-WSL: (TODO - haven't installed yet)
-
-Ubuntu 22.04:
-
-Install:
-
-```bash
-curl -fsSL https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo gpg --dearmor -o /usr/share/keyrings/elastic.gpg
-
-echo "deb [signed-by=/usr/share/keyrings/elastic.gpg] https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list
-
-sudo apt update
-sudo apt install elasticsearch
-```
-
-Configure:
-
-```text
-# Edit /etc/elasticsearch/elasticsearch.yml
-# Set the following:
-network.host: localhost
-```
-
-Run:
-
-```bash
-sudo systemctl start elasticsearch
-sudo systemctl enable elasticsearch
-```
-
-Verify:
-
-```bash
-curl -X GET 'http://localhost:9200'
-```
-
-Install Kibana (prototyping UI):
-
-```bash
-# Install
-sudo apt-get install kibana
-
-# Run
-sudo /bin/systemctl daemon-reload
-sudo /bin/systemctl enable kibana.service
-sudo systemctl start kibana.service
-
-# Access the Kibana Status Page:
-# http://localhost:5601/status
-
-# Access the query console UI:
-# Click Sidebar > Management > Dev Tools > Console
-# http://localhost:5601/
-```
-
-Extra reading on setting up Elasticsearch: 
-
-- [Guide 1](https://www.digitalocean.com/community/tutorials/how-to-install-and-configure-elasticsearch-on-ubuntu-22-04)
-- [Guide 2](https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html)
-- [Kibana](https://www.elastic.co/guide/en/kibana/current/deb.html)
-
-### (11) Run one or more applications:
-
-To start the HTTP API server,
-
-```bash
-cargo run --bin storyteller-web
-```
-
-Note that this compiles and runs the "development" binary. It's faster and easier to debug than the
-optimized "production" build. To build a fully optimized production release,
-run `cargo build --release --bin storyteller-web` . Note that this will take much longer.
-
-You may notice that this might not work right off the bat. You may need to specify some environment variables 
-and supply some development secrets.
-
-Secrets: 
-
-- Global secrets (ask Brandon for these): 
-  - Put `.env-secrets` in the root `storyteller-rust` directory.
-  
-- Per app secrets (ask Brandon for these):
-  - Put `storteller-web.development-secrets.env` in `crates/service/web/storyteller_web/config`
-  - Put `inference-job.development-secrets.env` in `crates/service/job/inference_job/config`
-  - Put `donwload-job.development-secrets.env` in `crates/service/job/download_job/config`
-
-**DO NOT CHECK THESE SECRET FILES INTO THE REPO!**
-
-Environment config: 
-
-You may need to set local development paths for your environment. You can set these in `.env-secrets` too,
-so that they won't be updated for other engineers.
-
-```
-# this should be the directory containing all of the "storyteller" projects:
-STORYTELLER_ROOT = "/home/tensor/code/storyteller" # change this
-
-# The following are entirely optional, but if you want to fine tune where each project lives, you may do so:
-STORYTELLER_FRONTEND = "/somewhere/else/storyteller-frontend"
-STORYTELLER_ML = "storyteller-ml"
-STORYTELLER_RUST = "/Users/bt/Development/storyteller-rust"
-```
-
-To download some ML models, run:
-
-```bash
-cargo run --bin download-job
-cargo run --bin tts-download-job
-```
-
-If you want to run both the HTTP API and the jobs, you'll need to run both processes in different
-terminals or tmux sessions.
-
-Note also that the configurations pointing to the ML monorepo must be set up for each application.
-Additionally, some development secrets may be needed (ask the team to share).
-
-To execute ML inference, run:
-
-```bash
-cargo run --bin inference-job
-cargo run --bin tts-inference-job
-```
-
-Again, note that the configurations pointing to the ML monorepo must be set up for each application.
-Additionally, some development secrets may be needed (ask the team to share).
-
-Seeding the Database with Some Values
--------------------------------------
-For voice designer you might want some pre-seeded audio that will hit the google cloud bucket, it will require env-secrets ask Brandon.
-cargo run --bin dev-database-seed -- --bucket
-
-Solutions to Common Setup Problems 
------------------------------------
-
-### [Fix] Sqlx Error when Performing Schema Migrations
-
-You might get this error message during migration,
-
-```
-Encountered unknown type for Mysql: enum
-thread 'main' panicked at 'internal error: entered unreachable code: Mysql only supports a closed set of types.
-                         If you ever see this error message please open an issue at https://github.com/diesel-rs/diesel 
-                         containing a dump of your schema definition.', diesel_cli/src/print_schema.rs:310:17
-note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
-```
-
-I haven't found the cause (it doesn't happen on newer installs), but the migrations appear to work regardless 
-of this error message. You can essentially ignore it.
-
-### [Fix] Can't connect to local MySQL after install on Ubuntu
-
-- If MySql in local dev can't be connected to, [reset the accounts](https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux).
-
-### API Server
-```
-cargo run --bin docs
-```
-This setups an api server with swagger docs to show you responses and requests with each associated endpoint
-It is run at http://127.0.0.1:8989/
-To obtain the json for the postman collection go to this url http://127.0.0.1:8989/api-docs/openapi.json
-We use this: `https://github.com/juhaku/utoipa`
-The documentation example is in: 
-`crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs`
-Write into the api doc to: 
-`crates/service/web/storyteller_web/src/api_doc.rs`

_docs/code_conventions.md

@@ -0,0 +1,26 @@
+Code Conventions
+================
+
+We use three languages at Artcraft: Rust, TypeScript, and Python.
+
+## Rust style guide
+
+We deviate from the official Rust style guide in a few places:
+
+- We use two spaces for indentation instead of four.
+- We typically keep one function or type definition per file, though this isn't a steadfast rule.
+- We use a Cargo workspace to share code within our monorepo.
+
+### Rust Preferred Libraries
+
+We use the following Rust libraries, though we are open to migrating to alternatives:
+
+- `actix-web` for our HTTP web server
+- `reqwest` for HTTP client requests
+- `sqlx` for SQL database operations
+- `tokio` for asynchronous programming
+
+## TypeScript style guide
+
+- We use two spaces for indentation instead of four.
+- We use `nx` for monorepo management

_docs/local_development_setup_guide.md

@@ -0,0 +1,395 @@
+Local Development Setup Guide (for bare metal)
+==============================================
+
+This document describes how to run the FakeYou infrastructure on your development 
+machine's bare metal (no containers).
+
+Setup instructions and development were tested and known to work on the following
+platforms:
+
+- Mac OS (Apple M2 silicon)
+- Ubuntu 22.04
+- Windows Subsystem for Linux (Ubuntu 22.04 distro) aka "WSL"
+
+The applications (binary targets) you might be interested in running include, but 
+are not limited to:
+
+- `storyteller-web`, the HTTP API server
+- `download-job`, which downloads models from the web and stores them in GCS
+- `inference-job`, which downloads models from GCS and runs inference with them
+
+Setup Instructions
+------------------
+
+### (1) Install Rust
+
+Install Rust [using this guide](https://www.rust-lang.org/learn/get-started). If it asks, 
+you'll want "stable" Rust, not "nightly" Rust. If it doesn't ask, it defaults to "stable".
+
+For Windows Subsystem for Linux, you'll want the WSL version, not the `.exe` file!
+
+### (2) Install the necessary libraries
+
+#### Mac
+
+On an up-to-date M1/M2 Mac, there should be no libraries you need to install! Any exceptions 
+to this assumption should be documented here.
+
+#### Ubuntu 22.04 and WSL:
+
+If you're on Ubuntu or WSL, you'll need to install some libraries to build the Rust code: 
+
+```bash
+# If you haven't run apt before, you'll need to fetch the package list:
+sudo apt update
+
+# Install the dependencies
+sudo apt install jq \
+  libmysqlclient-dev \
+  libsqlite3-dev \
+  pkgconf
+```
+
+(Note: jq is for combining mysql codegen outputs; pkgconf is if using openssl instead of rustls.)
+
+### (3) Install MySQL server
+
+#### Mac
+
+```bash
+# Install MySQL
+brew install mysql
+
+# Try to connect
+mysql -uroot
+```
+
+In the event of failure to connect, try the following steps:
+
+```bash
+# If connection fails, start service
+brew services start mysql
+
+# Try to connect again 
+# If this fails, reboot the machine, eg. using `sudo reboot now`
+mysql -uroot
+```
+
+#### Ubuntu 22.04 and WSL
+
+```bash
+# Install MySQL
+sudo apt install mysql-server
+
+# Try to connect 
+# The default password is typically "root".
+sudo mysql -u root -p
+````
+
+In the event of failure to connect, try the following steps:
+
+```bash
+# If it failed, you may need to start MySQL (typically only on WSL, not on native Ubuntu)
+sudo service mysql start
+
+# You may also have to change some socket file permissions (again, typically only on WSL)
+sudo chmod g+rx /var/run/mysqld
+sudo usermod -aG mysql $USER
+newgrp mysql
+
+# Try to connect again
+# The default password is typically "root".
+# If this fails, try rebooting. If it still fails, read some of the suggestions at the bottom 
+# of this README.
+sudo mysql -u root -p
+```
+
+### (4) Install a `storyteller` user and table in MySQL
+
+First, connect to MySQL:
+
+```bash
+# If the following command asks for a password, the password is typically "root"
+sudo mysql -u root -p
+```
+
+Once in MySQL, run the following to set up the database and user accounts:
+
+```mysql
+use mysql;
+CREATE DATABASE storyteller;
+CREATE USER 'storyteller'@'localhost' IDENTIFIED BY 'password';
+GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost';
+```
+
+Then verify access with `./dev_mysql_connect.sh`
+
+### (5) Install Diesel CLI (for MySQL migrations):
+
+In order to run MySQL database migrations, you'll need to install the migration tool we use:
+
+```bash
+cargo install diesel_cli \
+  --no-default-features \
+  --features mysql,sqlite
+```
+
+In older Macs, there [have been some issues with Diesel CLI](https://github.com/diesel-rs/diesel/issues/2605)
+that require a few extra dependencies to be installed:
+
+```bash
+# If you're on Mac and the above command didn't work, run the following and then retry:
+brew install libpq
+```
+
+### (6) Run the pending database migrations:
+
+You're ready to run the migrations:
+
+```bash
+diesel migration run
+```
+
+You might get a scary message about `"Encountered unknown type for Mysql: enum"` -- you can safely ignore this 
+error if you see it in isolation. It doesn't impact the migrations whatsoever. (See notes at the bottom of 
+the README for details.)
+
+### (7) (optional) Install the SQLx CLI (if doing database development)
+
+(You can probably skip this step unless you're a backend engineer.) 
+
+If (and only if) you're doing Rust service development that changes the MySQL queries, then you should install 
+SQLx utilities. This tool is used to cache the column types and query plans for CI, as SQLx statically verifies 
+raw MySQL queries against the schema and Rust types.
+
+We'll be using diesel to manage the migrations, but sqlx within the app to actually perform queries.
+Diesel is an immature ORM, which isn't a good tech bet, so we use sqlx as at-compile-time
+typesafe SQL. 
+
+(TODO: Reevaluate this assessment for 2023-2024 and beyond. Maybe we do want Diesel now.)
+
+```bash
+# NB(bt,2023-10-04): "--no-default-features" not working
+# cargo install sqlx-cli --no-default-features --features rustls,mysql,sqlite
+cargo install sqlx-cli --features rustls,mysql,sqlite
+```
+
+### (8) Install hosts file:
+
+(This is the same for Linux, Mac, and WSL.)
+
+If you're developing against the frontend, it'll target development domains (eg. `dev.fakeyou.com`) instead 
+of `127.0.0.1` or `localhost`. You can make your machine route domains to localhost by editing your hosts 
+file (located at `/etc/hosts`) to include the following configuration lines:
+
+```
+# 1) Edit hosts file with your editor of choice
+#
+#  - sudo vi /etc/hosts
+#  - sudo nano /etc/hosts
+#
+# 2) Then paste:
+#
+
+127.0.0.1    dev.fakeyou.com
+127.0.0.1    api.dev.fakeyou.com
+127.0.0.1    devproxy.fakeyou.com
+
+127.0.0.1    dev.storyteller.ai
+127.0.0.1    api.dev.storyteller.ai
+127.0.0.1    devproxy.storyteller.ai
+```
+
+### (9) Install Redis
+
+#### Mac:
+
+```bash
+# Install Redis
+brew install redis
+
+# Start the service:
+brew services start redis
+```
+
+#### Ubuntu 22.04 and WSL
+
+```
+# Install Redis
+sudo apt install redis
+
+# Start the service (typically only needed on WSL):
+sudo service redis-server start
+```
+
+### (10) Install Elasticsearch (optional)
+
+(You can most likely skip this step.)
+
+Mac: (TODO - haven't installed yet)
+
+WSL: (TODO - haven't installed yet)
+
+Ubuntu 22.04:
+
+Install:
+
+```bash
+curl -fsSL https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo gpg --dearmor -o /usr/share/keyrings/elastic.gpg
+
+echo "deb [signed-by=/usr/share/keyrings/elastic.gpg] https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list
+
+sudo apt update
+sudo apt install elasticsearch
+```
+
+Configure:
+
+```text
+# Edit /etc/elasticsearch/elasticsearch.yml
+# Set the following:
+network.host: localhost
+```
+
+Run:
+
+```bash
+sudo systemctl start elasticsearch
+sudo systemctl enable elasticsearch
+```
+
+Verify:
+
+```bash
+curl -X GET 'http://localhost:9200'
+```
+
+Install Kibana (prototyping UI):
+
+```bash
+# Install
+sudo apt-get install kibana
+
+# Run
+sudo /bin/systemctl daemon-reload
+sudo /bin/systemctl enable kibana.service
+sudo systemctl start kibana.service
+
+# Access the Kibana Status Page:
+# http://localhost:5601/status
+
+# Access the query console UI:
+# Click Sidebar > Management > Dev Tools > Console
+# http://localhost:5601/
+```
+
+Extra reading on setting up Elasticsearch: 
+
+- [Guide 1](https://www.digitalocean.com/community/tutorials/how-to-install-and-configure-elasticsearch-on-ubuntu-22-04)
+- [Guide 2](https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html)
+- [Kibana](https://www.elastic.co/guide/en/kibana/current/deb.html)
+
+### (11) Run one or more applications:
+
+To start the HTTP API server,
+
+```bash
+cargo run --bin storyteller-web
+```
+
+Note that this compiles and runs the "development" binary. It's faster and easier to debug than the
+optimized "production" build. To build a fully optimized production release,
+run `cargo build --release --bin storyteller-web` . Note that this will take much longer.
+
+You may notice that this might not work right off the bat. You may need to specify some environment variables 
+and supply some development secrets.
+
+Secrets: 
+
+- Global secrets (ask Brandon for these): 
+  - Put `.env-secrets` in the root `storyteller-rust` directory.
+  
+- Per app secrets (ask Brandon for these):
+  - Put `storteller-web.development-secrets.env` in `crates/service/web/storyteller_web/config`
+  - Put `inference-job.development-secrets.env` in `crates/service/job/inference_job/config`
+  - Put `donwload-job.development-secrets.env` in `crates/service/job/download_job/config`
+
+**DO NOT CHECK THESE SECRET FILES INTO THE REPO!**
+
+Environment config: 
+
+You may need to set local development paths for your environment. You can set these in `.env-secrets` too,
+so that they won't be updated for other engineers.
+
+```
+# this should be the directory containing all of the "storyteller" projects:
+STORYTELLER_ROOT = "/home/tensor/code/storyteller" # change this
+
+# The following are entirely optional, but if you want to fine tune where each project lives, you may do so:
+STORYTELLER_FRONTEND = "/somewhere/else/storyteller-frontend"
+STORYTELLER_ML = "storyteller-ml"
+STORYTELLER_RUST = "/Users/bt/Development/storyteller-rust"
+```
+
+To download some ML models, run:
+
+```bash
+cargo run --bin download-job
+cargo run --bin tts-download-job
+```
+
+If you want to run both the HTTP API and the jobs, you'll need to run both processes in different
+terminals or tmux sessions.
+
+Note also that the configurations pointing to the ML monorepo must be set up for each application.
+Additionally, some development secrets may be needed (ask the team to share).
+
+To execute ML inference, run:
+
+```bash
+cargo run --bin inference-job
+cargo run --bin tts-inference-job
+```
+
+Again, note that the configurations pointing to the ML monorepo must be set up for each application.
+Additionally, some development secrets may be needed (ask the team to share).
+
+Seeding the Database with Some Values
+-------------------------------------
+For voice designer you might want some pre-seeded audio that will hit the google cloud bucket, it will require env-secrets ask Brandon.
+cargo run --bin dev-database-seed -- --bucket
+
+Solutions to Common Setup Problems 
+-----------------------------------
+
+### [Fix] Sqlx Error when Performing Schema Migrations
+
+You might get this error message during migration,
+
+```
+Encountered unknown type for Mysql: enum
+thread 'main' panicked at 'internal error: entered unreachable code: Mysql only supports a closed set of types.
+                         If you ever see this error message please open an issue at https://github.com/diesel-rs/diesel 
+                         containing a dump of your schema definition.', diesel_cli/src/print_schema.rs:310:17
+note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
+```
+
+I haven't found the cause (it doesn't happen on newer installs), but the migrations appear to work regardless 
+of this error message. You can essentially ignore it.
+
+### [Fix] Can't connect to local MySQL after install on Ubuntu
+
+- If MySql in local dev can't be connected to, [reset the accounts](https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux).
+
+### API Server
+```
+cargo run --bin docs
+```
+This setups an api server with swagger docs to show you responses and requests with each associated endpoint
+It is run at http://127.0.0.1:8989/
+To obtain the json for the postman collection go to this url http://127.0.0.1:8989/api-docs/openapi.json
+We use this: `https://github.com/juhaku/utoipa`
+The documentation example is in: 
+`crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs`
+Write into the api doc to: 
+`crates/service/web/storyteller_web/src/api_doc.rs`

_sql/artcraft_migrations/20250708000000_create_tasks_table.sql

@@ -0,0 +1,56 @@
+-- noinspection SqlDialectInspectionForFile
+-- Migration history:
+--   tasks_v1.sqlite - initial version
+--   tasks_v2.sqlite - added model_type (nullable)
+
+CREATE TABLE tasks (
+    -- Task auto-incrementing primary key.
+    -- We don't use this.
+    id TEXT NOT NULL PRIMARY KEY,
+
+    -- TaskStatus enum
+    -- This is how the job system manages job states.
+    -- e.g. pending, started, complete_success, dead, etc.
+    task_status TEXT NOT NULL,
+
+    -- TaskType enum
+    -- e.g. image_generation, video_generation, etc.
+    task_type TEXT NOT NULL,
+
+    -- TaskModelType enum
+    -- e.g. flux_1_dev, veo_2, etc.
+    model_type TEXT,
+
+    -- GenerationProvider enum
+    -- Together with `provider_job_id`, this creates a foreign key lookup to the job.
+    -- e.g. 'artcraft', 'fal', 'sora', etc.
+    provider TEXT,
+
+    -- The primary key for the job in the provider's system.
+    provider_job_id TEXT,
+
+    -- OPTIONAL.
+    -- Tell the job system which caller initiated the task.
+    -- e.g. 'canvas'
+    frontend_caller TEXT,
+
+    -- OPTIONAL.
+    -- An arbitrary opaque identifier that frontend can set.
+    frontend_subscriber_id TEXT,
+
+    -- OPTIONAL.
+    -- An opaque JSON payload set by the frontend subscriber that
+    -- will be re-emitted back to the frontend.
+    frontend_subscriber_payload TEXT,
+
+    created_at INTEGER NOT NULL DEFAULT (unixepoch('now')),
+    updated_at INTEGER NOT NULL DEFAULT (unixepoch('now')),
+    completed_at INTEGER DEFAULT NULL
+);
+
+-- Unique Indices
+CREATE UNIQUE INDEX idx_tasks_on_provider_job_id ON tasks(provider_job_id);
+
+-- Indices
+CREATE INDEX idx_tasks_on_status ON tasks(task_status);
+CREATE INDEX idx_tasks_on_provider ON tasks(provider);

_sql/migrations/2025-06-02-085011_jobs_add_third_party_columns/down.sql

@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE generic_inference_jobs DROP COLUMN maybe_external_third_party_id;
+ALTER TABLE generic_inference_jobs DROP COLUMN maybe_external_third_party;

_sql/migrations/2025-06-02-085011_jobs_add_third_party_columns/up.sql

@@ -0,0 +1,9 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE generic_inference_jobs
+ADD COLUMN maybe_external_third_party VARCHAR(16) DEFAULT NULL;
+
+ALTER TABLE generic_inference_jobs
+ADD COLUMN maybe_external_third_party_id VARCHAR(64) DEFAULT NULL;

_sql/migrations/2025-06-02-085015_jobs_add_third_party_indices/down.sql

@@ -0,0 +1,9 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+DROP INDEX `index_maybe_external_third_party_id`
+    ON `generic_inference_jobs`;
+
+DROP INDEX `index_maybe_external_third_party_and_id`
+    ON `generic_inference_jobs`;

_sql/migrations/2025-06-02-085015_jobs_add_third_party_indices/up.sql

@@ -0,0 +1,11 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE generic_inference_jobs
+    ADD UNIQUE INDEX index_maybe_external_third_party_and_id (maybe_external_third_party, maybe_external_third_party_id),
+    ALGORITHM=INPLACE, LOCK=NONE;
+
+ALTER TABLE generic_inference_jobs
+    ADD INDEX index_maybe_external_third_party_id (maybe_external_third_party_id),
+    ALGORITHM=INPLACE, LOCK=NONE;

_sql/migrations/2025-07-23-031631_alter_prompts_add_model/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE prompts DROP COLUMN maybe_model_type;

_sql/migrations/2025-07-23-031631_alter_prompts_add_model/up.sql

@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE prompts
+    ADD COLUMN maybe_model_type VARCHAR(32) DEFAULT NULL;

_sql/migrations/2025-07-23-031650_alter_prompts_add_provider/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE prompts DROP COLUMN maybe_generation_provider;

_sql/migrations/2025-07-23-031650_alter_prompts_add_provider/up.sql

@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE prompts
+    ADD COLUMN maybe_generation_provider VARCHAR(16) DEFAULT NULL;

_sql/migrations/2025-07-23-031720_alter_prompts_add_maybe_anonymous_visitor_token/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE prompts DROP COLUMN maybe_anonymous_visitor_token;

_sql/migrations/2025-07-23-031720_alter_prompts_add_maybe_anonymous_visitor_token/up.sql

@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE prompts
+    ADD COLUMN maybe_anonymous_visitor_token VARCHAR(32) DEFAULT NULL;

_sql/migrations/2025-07-23-200905_prompt_context_items/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+DROP TABLE prompt_context_items;

_sql/migrations/2025-07-23-200905_prompt_context_items/up.sql

@@ -0,0 +1,30 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- An n:m join table between prompts and media_files.
+-- This associates mostly image media files with prompts, though we could conceivably
+-- also store 3D sets, etc.
+CREATE TABLE prompt_context_items (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- The semantic type of context.
+  context_semantic_type VARCHAR(16) NOT NULL,
+
+  -- Foreign key, many-to-many
+  -- There is no uniqueness constraint for (prompt_token, media_token) because a user
+  -- might attach many duplicate images to the same prompt.
+  prompt_token VARCHAR(32) NOT NULL,
+
+  -- Foreign key to the media token
+  -- There is no uniqueness constraint for (prompt_token, media_token) because a user
+  -- might attach many duplicate images to the same prompt.
+  media_token VARCHAR(32) NOT NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  KEY index_prompt_token (prompt_token),
+  KEY index_media_token (media_token)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_sql/migrations/2025-07-23-211516_alter_generic_inference_jobs_add_maybe_prompt_token/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE generic_inference_jobs DROP COLUMN maybe_prompt_token;

_sql/migrations/2025-07-23-211516_alter_generic_inference_jobs_add_maybe_prompt_token/up.sql

@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE generic_inference_jobs
+    ADD COLUMN maybe_prompt_token VARCHAR(32) DEFAULT NULL;

_sql/migrations/2025-07-23-212848_alter_media_files_add_maybe_generation_provider/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE media_files DROP COLUMN maybe_generation_provider;

_sql/migrations/2025-07-23-212848_alter_media_files_add_maybe_generation_provider/up.sql

@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE media_files
+    ADD COLUMN maybe_generation_provider VARCHAR(16) DEFAULT NULL;

_sql/migrations_squashed/generic_inference_jobs.sql

@@ -21,6 +21,8 @@ CREATE TABLE generic_inference_jobs (
   -- The new enum for the type of job, which will eventually replace `inference_category` and `maybe_model_type`.
   -- This isn't fully supported yet in the inference-job, but we'll start populating it and adding logic around it.
   -- We'll start this out as nullable, then migrate the old rows with a default column value.
+  --
+  -- Fal jobs have the type `fal_queue`.
   job_type VARCHAR(32) DEFAULT NULL,
 
   -- There is an index on this column.
@@ -39,8 +41,20 @@ CREATE TABLE generic_inference_jobs (
   -- Examples (may not be up to date):
   --  * text_to_speech
   --  * voice_conversion
+  --  * image_generation
+  --  * video_generation  
+  --  * background_removal
   inference_category VARCHAR(32) NOT NULL,
 
+  -- If the job is externally enqueued, this is the third-party service
+  -- responsible for the job. Typically, we won't run an inference job for
+  -- jobs being run on a third party.
+  maybe_external_third_party VARCHAR(16) DEFAULT NULL,
+
+  -- If the job is externally enqueued, this is the ID the third-party
+  -- service uses.
+  maybe_external_third_party_id VARCHAR(64) DEFAULT NULL,
+
   -- There is an index on this column.
   -- NB: See notes on inference_category. This is becoming a problematic field. We're using
   -- this to load the container spin-up arguments and dependencies, but the former is being
@@ -91,6 +105,11 @@ CREATE TABLE generic_inference_jobs (
   -- And furthermore, not all download URLs have a cover image.
   maybe_cover_image_media_file_token VARCHAR(32) DEFAULT NULL,
 
+  -- If we pre-constructed a prompt, this will be copied to the resultant media generated.
+  -- This is especially important for Fal-related jobs and endpoints. We need this for keeping
+  -- track of all the prompt details, context images, etc.
+  maybe_prompt_token VARCHAR(30) DEFAULT NULL,
+
   -- A migration flag we can control from the backend API service which will tell
   -- the worker where to store the results. If false, GCP. If true, AWS.
   store_in_aws BOOLEAN NOT NULL DEFAULT FALSE,
@@ -287,11 +306,13 @@ CREATE TABLE generic_inference_jobs (
   PRIMARY KEY (id),
   UNIQUE KEY (token),
   UNIQUE KEY (uuid_idempotency_token),
+  UNIQUE KEY (maybe_external_third_party, maybe_external_third_party_id),
   KEY index_job_type (job_type),
   KEY index_product_category (product_category),
   KEY index_inference_category (inference_category),
   KEY index_maybe_model_type_and_maybe_model_token (maybe_model_type, maybe_model_token),
   KEY index_maybe_model_type (maybe_model_type),
+  KEY index_maybe_external_third_party_id (maybe_external_third_party_id),
   KEY fk_maybe_model_token (maybe_model_token),
   KEY fk_maybe_input_source_token (maybe_input_source_token),
   KEY fk_maybe_input_source_token_and_type (maybe_input_source_token, maybe_input_source_token_type),

_sql/migrations_squashed/media_files.sql

@@ -61,6 +61,9 @@ CREATE TABLE media_files (
   --   * 'workflow' (DEPRECATED)
   origin_product_category VARCHAR(16) NOT NULL DEFAULT "unknown",
 
+  -- What service provided the generation (if any).
+  maybe_generation_provider VARCHAR(16) DEFAULT NULL,
+
   -- For inference that can be tied back to a model, the type of model.
   -- There are multiple indices on this column!
   -- DO NOT EXPOSE THIS TO USERS VIA THE API, as we may leak secrets in doing so.

_sql/migrations_squashed/prompts/batch_generations.sql

@@ -15,9 +15,9 @@ CREATE TABLE batch_generations (
 
   -- The type of record this is attached to
   -- Currently supported values:
-  --   * 'model_weight'
+  --   * 'model_weight' (deprecated)
+  --   * 'comment' (deprecated)
   --   * 'media_file'
-  --   * 'comment'
   entity_type VARCHAR(32) NOT NULL,
 
   -- The token for the record.
@@ -37,6 +37,4 @@ CREATE TABLE batch_generations (
   -- NB: Realistically, we probably don't need this index until we have a lot of entity types, and the entity_token
   -- entropy is probably sufficient enough as-is.
   -- KEY index_entity_type_and_token (entity_type, entity_token),
-
-
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_sql/migrations_squashed/prompts/prompt_context_items.sql

@@ -0,0 +1,30 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- An n:m join table between prompts and media_files.
+-- This associates mostly image media files with prompts, though we could conceivably
+-- also store 3D sets, etc.
+CREATE TABLE prompt_context_items (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Foreign key, many-to-many
+  -- There is no uniqueness constraint for (prompt_token, media_token) because a user
+  -- might attach many duplicate images to the same prompt.
+  prompt_token VARCHAR(32) NOT NULL,
+
+  -- Foreign key to the media token
+  -- There is no uniqueness constraint for (prompt_token, media_token) because a user
+  -- might attach many duplicate images to the same prompt.
+  media_token VARCHAR(32) NOT NULL,
+
+  -- The semantic type of context.
+  context_semantic_type VARCHAR(16) NOT NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  KEY index_prompt_token (prompt_token),
+  KEY index_media_token (media_token)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_sql/migrations_squashed/prompts/prompts.sql

@@ -19,13 +19,25 @@ CREATE TABLE prompts (
   -- Original product to which the prompt applied.
   -- Prompts may or may not be compatible across products.
   -- Possible values:
-  --  * 'stable_diffusion'
-  --  * 'comfy_ui'
+  --  * 'artcraft_app'
+  --  * 'stable_diffusion' (legacy)
+  --  * 'comfy_ui' (legacy)
   prompt_type VARCHAR(16) NOT NULL,
 
-  -- Whomever created the prompt
+  -- What type of model was used to generate the result.
+  -- NB: We're conflating prompts with outputs, but in a
+  -- sense the model is inextricably part of the prompt.
+  maybe_model_type VARCHAR(32) DEFAULT NULL,
+
+  -- What service provided the generation.
+  maybe_generation_provider VARCHAR(16) NOT NULL,
+
+  -- Whoever created the prompt
   maybe_creator_user_token VARCHAR(32) DEFAULT NULL,
 
+  -- Whoever created the prompt (anonymous user tracking)
+  maybe_anonymous_visitor_token VARCHAR(32) DEFAULT NULL,
+
   -- ========== PROMPT DETAILS ==========
 
   -- The positive prompt (optional)

crates/_types/composite_identifiers/Cargo.toml

@@ -1,39 +0,0 @@
-[package]
-name = "composite_identifiers"
-edition = "2018"
-version = "0.0.1"
-authors = [
-    "Brandon Thomas <bt@brand.io>",
-    "Brandon Thomas <echelon@gmail.com>",
-]
-publish = false
-
-[lib]
-name = "composite_identifiers"
-path = "src/lib.rs"
-
-[package.metadata.cargo-udeps.ignore]
-normal = ["workspace-build-acceleration"]
-
-[dependencies]
-# Internal
-enums = { path = "../enums" }
-tokens = { path = "../tokens" }
-
-# Workspace hack (faster compile times)
-workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
-
-# Serialization
-serde = "1.0.125"
-serde_derive = "1.0.125"
-
-sqlx = { version = "0.7.2", features = [ "mysql", "sqlite", "runtime-tokio-native-tls", "chrono" ] }
-sqlx-core = { version = "0.7.2" }
-sqlx-mysql= { version = "0.7.2" }
-
-utoipa = { version = "4", features = ["chrono","actix_extras"] }
-
-[dev-dependencies]
-
-# Test serialization
-serde_json = "1.0.108"

crates/_types/enums/Cargo.toml

@@ -1,41 +0,0 @@
-[package]
-name = "enums"
-edition = "2018"
-version = "0.0.1"
-authors = [
-    "Brandon Thomas <bt@brand.io>",
-    "Brandon Thomas <echelon@gmail.com>",
-]
-publish = false
-
-[lib]
-name = "enums"
-path = "src/lib.rs"
-
-[package.metadata.cargo-udeps.ignore]
-normal = ["workspace-build-acceleration"]
-
-[dependencies]
-# Internal
-
-# Workspace hack (faster compile times)
-workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
-
-# Serialization
-serde = "1.0.125"
-serde_derive = "1.0.125"
-
-sqlx = { version = "0.7.2", features = [ "mysql", "sqlite", "runtime-tokio-native-tls", "chrono" ] }
-sqlx-core = { version = "0.7.2" }
-sqlx-mysql= { version = "0.7.2" }
-
-utoipa = { version = "4", features = ["chrono","actix_extras"] }
-
-[dev-dependencies]
-
-# Enum utilities
-strum = { version = "0.26", features = ["derive"] }
-
-# Test serialization
-serde_json = "1.0.108"
-toml = "0.8.14"

crates/_types/enums/src/by_table/generic_inference_jobs/mod.rs

@@ -1,7 +0,0 @@
-pub mod frontend_failure_category;
-pub mod inference_category;
-pub mod inference_input_source_token_type;
-pub mod inference_job_product_category;
-pub mod inference_job_type;
-pub mod inference_model_type;
-pub mod inference_result_type;

crates/_types/enums/src/by_table/mod.rs

@@ -1,31 +0,0 @@
-// ===== MySql =====
-pub mod audit_logs;
-pub mod batch_generations;
-pub mod beta_keys;
-pub mod comments;
-pub mod email_sender_jobs;
-pub mod entity_stats;
-pub mod featured_items;
-pub mod generic_download_jobs;
-pub mod generic_inference_jobs;
-pub mod generic_synthetic_ids;
-pub mod media_files;
-pub mod media_uploads;
-pub mod model_categories;
-pub mod model_weights;
-pub mod prompts;
-pub mod tag_uses;
-pub mod trending_model_analytics;
-pub mod tts_models;
-pub mod usages;
-pub mod user_bookmarks;
-pub mod user_ratings;
-pub mod users;
-pub mod voice_conversion_models;
-pub mod voice_conversion_results;
-pub mod zs_voices;
-
-// ===== Sqlite =====
-pub mod tts_render_tasks;
-pub mod web_rendition_targets;
-pub mod web_scraping_targets;

crates/_types/enums/src/common/mod.rs

@@ -1,6 +0,0 @@
-pub mod job_status;
-pub mod job_status_plus;
-pub mod sqlite;
-pub mod visibility;
-pub mod vocoder_type;
-pub mod view_as;

crates/_types/enums/src/lib.rs

@@ -1,37 +0,0 @@
-//! enums
-//!
-//! The purpose of this library is to have a strongly-typed MySQL enum-type wrapper.
-//! This should also work for CHAR/VARCHAR fields that work similarly to enums (typically
-//! as part of a composite key)
-//!
-//! These types should also be friendly for API usage in JSON payloads.
-//!
-//! In the future this should be *CODEGEN DRIVEN* and should get checked into source control.
-//!
-
-// Never allow these
-#![forbid(private_bounds)]
-#![forbid(private_interfaces)]
-#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
-
-// Okay to toggle
-#![forbid(unreachable_patterns)]
-#![forbid(unused_imports)]
-#![forbid(unused_mut)]
-#![forbid(unused_variables)]
-
-// Always allow
-#![allow(dead_code)]
-#![allow(non_snake_case)]
-
-#[macro_use] extern crate serde_derive;
-
-#[macro_use]
-mod macros;
-
-#[cfg(test)] pub mod test_helpers;
-
-pub mod by_table;
-pub mod common;
-pub mod no_table;
-pub mod traits;

crates/_types/enums/src/traits/mod.rs

@@ -1 +0,0 @@
-pub mod mysql_from_row;

crates/_types/enums_public/Cargo.toml

@@ -1,38 +0,0 @@
-[package]
-name = "enums_public"
-edition = "2018"
-version = "0.0.1"
-authors = [
-    "Brandon Thomas <bt@brand.io>",
-    "Brandon Thomas <echelon@gmail.com>",
-]
-publish = false
-
-[lib]
-name = "enums_public"
-path = "src/lib.rs"
-
-[package.metadata.cargo-udeps.ignore]
-normal = ["workspace-build-acceleration"]
-
-[dependencies]
-# Workspace hack (faster compile times)
-workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
-
-# Internal
-enums = { path = "../enums" }
-
-# Serialization
-serde = "1.0.125"
-serde_derive = "1.0.125"
-
-utoipa = { version = "4", features = ["chrono","actix_extras"] }
-
-[dev-dependencies]
-
-# Enum utilities
-strum = { version = "0.26", features = ["derive"] }
-
-# Test serialization
-serde_json = "1.0.108"
-toml = "0.8.14"

crates/_types/reusable_types/Cargo.toml

@@ -1,39 +0,0 @@
-[package]
-name = "reusable_types"
-publish = false
-edition = "2018"
-version = "0.0.1"
-authors = [
-    "Brandon Thomas <bt@brand.io>",
-    "Brandon Thomas <echelon@gmail.com>",
-]
-
-[lib]
-name = "reusable_types"
-path = "src/lib.rs"
-
-[package.metadata.cargo-udeps.ignore]
-normal = ["workspace-build-acceleration"]
-
-[dependencies]
-
-# Internal
-# (none)
-
-# Workspace hack (faster compile times)
-workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
-
-# External
-chrono = { version = "0.4.22", features = ["serde"] }
-
-# Serialization
-serde = "1.0.125"
-serde_derive = "1.0.125"
-
-# MySql Database
-sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] }
-
-[dev-dependencies]
-# Test serialization
-#serde_json = "1.0.64"
-toml = "0.8.14"

crates/_types/tokens/Cargo.toml

@@ -1,48 +0,0 @@
-[package]
-name = "tokens"
-edition = "2018"
-version = "0.0.1"
-authors = [
-    "Brandon Thomas <bt@brand.io>",
-    "Brandon Thomas <echelon@gmail.com>",
-]
-publish = false
-
-[lib]
-name = "tokens"
-path = "src/lib.rs"
-
-[package.metadata.cargo-udeps.ignore]
-normal = ["workspace-build-acceleration"]
-
-[dependencies]
-
-# Internal
-# (none)
-
-# Workspace hack (faster compile times)
-workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" }
-
-# Serialization
-serde = "1.0.125"
-serde_derive = "1.0.125"
-
-# Random
-rand = {  version = "0.8.5", features = ["std_rng"] }
-
-once_cell = { version = "1.18.0", features = [] }
-
-sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] }
-sqlx-core = { version = "0.7.2" }
-sqlx-mysql= { version = "0.7.2" }
-
-utoipa = { version = "4", features = ["chrono","actix_extras"] }
-
-[dev-dependencies]
-
-# Enum utilities
-strum = { version = "0.26", features = ["derive"] }
-
-# Test serialization
-serde_json = "1.0.108"
-toml = "0.8.14"

crates/_types/tokens/src/tokens/api_tokens_external.rs

@@ -1,14 +0,0 @@
-use std::fmt::Debug;
-
-use serde::Deserialize;
-use serde::Serialize;
-
-use crate::prefixes::LegacyTokenPrefix;
-
-/// External-facing key for the `api_tokens` table.
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize)]
-#[sqlx(transparent)]
-pub struct ApiTokenExternal(pub String);
-
-impl_string_token!(ApiTokenExternal);
-impl_crockford_generator!(ApiTokenExternal, 32usize, LegacyTokenPrefix::ApiTokenExternal, CrockfordUpper);

crates/_types/tokens/src/tokens/api_tokens_internal.rs

@@ -1,14 +0,0 @@
-use std::fmt::Debug;
-
-use serde::Deserialize;
-use serde::Serialize;
-
-use crate::prefixes::LegacyTokenPrefix;
-
-/// Internal token for the `api_tokens` table.
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize)]
-#[sqlx(transparent)]
-pub struct ApiTokenInternal(pub String);
-
-impl_string_token!(ApiTokenInternal);
-impl_crockford_generator!(ApiTokenInternal, 32usize, LegacyTokenPrefix::ApiTokenInternal, CrockfordLower);

crates/_types/tokens/src/tokens/audit_logs.rs

@@ -1,14 +0,0 @@
-use std::fmt::Debug;
-
-use serde::Deserialize;
-use serde::Serialize;
-
-use crate::prefixes::TokenPrefix;
-
-/// The primary key for Audit Logs.
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize)]
-#[sqlx(transparent)]
-pub struct AuditLogToken(pub String);
-
-impl_string_token!(AuditLogToken);
-impl_crockford_generator!(AuditLogToken, 32usize, TokenPrefix::AuditLog, CrockfordLower);

crates/_types/tokens/src/tokens/batch_generations.rs

@@ -1,16 +0,0 @@
-use std::fmt::Debug;
-
-use serde::Deserialize;
-use serde::Serialize;
-use utoipa::ToSchema;
-
-use crate::prefixes::TokenPrefix;
-
-/// The primary key for Audit Logs.
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize, ToSchema)]
-#[sqlx(transparent)]
-pub struct BatchGenerationToken(pub String);
-
-impl_crockford_generator!(BatchGenerationToken, 32usize, TokenPrefix::BatchGeneration, CrockfordLower);
-impl_mysql_token_from_row!(BatchGenerationToken);
-impl_string_token!(BatchGenerationToken);

crates/_types/tokens/src/tokens/beta_keys.rs

@@ -1,16 +0,0 @@
-use std::fmt::Debug;
-
-use serde::Deserialize;
-use serde::Serialize;
-use utoipa::ToSchema;
-
-use crate::prefixes::TokenPrefix;
-
-/// The primary key for Audit Logs.
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize, ToSchema)]
-#[sqlx(transparent)]
-pub struct BetaKeyToken(pub String);
-
-impl_crockford_generator!(BetaKeyToken, 32usize, TokenPrefix::BetaKey, CrockfordLower);
-impl_mysql_token_from_row!(BetaKeyToken);
-impl_string_token!(BetaKeyToken);

crates/_types/tokens/src/tokens/browser_session_logs.rs

@@ -1,16 +0,0 @@
-use std::fmt::Debug;
-
-use serde::Deserialize;
-use serde::Serialize;
-use utoipa::ToSchema;
-
-use crate::prefixes::TokenPrefix;
-
-/// The primary key for Audit Logs.
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize, ToSchema)]
-#[sqlx(transparent)]
-pub struct BrowserSessionLogToken(pub String);
-
-impl_crockford_generator!(BrowserSessionLogToken, 32usize, TokenPrefix::BrowserSessionLog, CrockfordLower);
-impl_mysql_token_from_row!(BrowserSessionLogToken);
-impl_string_token!(BrowserSessionLogToken);

crates/_types/tokens/src/tokens/comments.rs

@@ -1,15 +0,0 @@
-use std::fmt::Debug;
-
-use serde::Deserialize;
-use serde::Serialize;
-use utoipa::ToSchema;
-
-use crate::prefixes::TokenPrefix;
-
-/// The primary key for Audit Logs.
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize, ToSchema)]
-#[sqlx(transparent)]
-pub struct CommentToken(pub String);
-
-impl_string_token!(CommentToken);
-impl_crockford_generator!(CommentToken, 32usize, TokenPrefix::Comment, CrockfordLower);

crates/_types/tokens/src/tokens/email_sender_job_token.rs

@@ -1,14 +0,0 @@
-use std::fmt::Debug;
-
-use serde::Deserialize;
-use serde::Serialize;
-
-use crate::prefixes::TokenPrefix;
-
-/// The primary key for email_sender_jobs
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize)]
-#[sqlx(transparent)]
-pub struct EmailSenderJobToken(pub String);
-
-impl_string_token!(EmailSenderJobToken);
-impl_crockford_generator!(EmailSenderJobToken, 32usize, TokenPrefix::EmailSenderJob, CrockfordLower);

crates/_types/tokens/src/tokens/firehose_entries.rs

@@ -1,14 +0,0 @@
-use std::fmt::Debug;
-
-use serde::Deserialize;
-use serde::Serialize;
-
-use crate::prefixes::LegacyTokenPrefix;
-
-/// Primary key for the `firehose_entries` table.
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize)]
-#[sqlx(transparent)]
-pub struct FirehoseEntryToken(pub String);
-
-impl_string_token!(FirehoseEntryToken);
-impl_crockford_generator!(FirehoseEntryToken, 32usize, LegacyTokenPrefix::FirehoseEntry, CrockfordLower);

crates/_types/tokens/src/tokens/generic_download_jobs.rs

@@ -1,14 +0,0 @@
-use std::fmt::Debug;
-
-use serde::Deserialize;
-use serde::Serialize;
-
-use crate::prefixes::TokenPrefix;
-
-/// The primary key for "generic" inference jobs.
-#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)]
-#[sqlx(transparent)]
-pub struct DownloadJobToken(String);
-
-impl_string_token!(DownloadJobToken);
-impl_crockford_generator!(DownloadJobToken, 32usize, TokenPrefix::DownloadJob, CrockfordLower);

crates/_types/tokens/src/tokens/generic_inference_jobs.rs

@@ -1,16 +0,0 @@
-use std::fmt::Debug;
-
-use serde::Deserialize;
-use serde::Serialize;
-use utoipa::ToSchema;
-
-use crate::prefixes::TokenPrefix;
-
-/// The primary key for "generic" inference jobs.
-#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize, Default, ToSchema)]
-#[sqlx(transparent)]
-pub struct InferenceJobToken(String);
-
-impl_crockford_generator!(InferenceJobToken, 32usize, TokenPrefix::InferenceJob, CrockfordLower);
-impl_mysql_token_from_row!(InferenceJobToken);
-impl_string_token!(InferenceJobToken);