TurfAI Deployment Strategy & Implementation Guide

npm install -g pm2

mkdir -p dev/scripts

module.exports = {
  apps: [
    // DMS - Strapi CMS (Port 1337)
    {
      name: 'dms',
      cwd: './dms',
      script: 'npm',
      args: 'run develop',
      env: {
        PORT: 1337,
        NODE_ENV: 'development',
        DATABASE_HOST: 'localhost',
        DATABASE_PORT: 5432,
        DATABASE_NAME: 'turfai_dms',
        DATABASE_USERNAME: 'postgres',
        DATABASE_PASSWORD: 'postgres',
        GCS_BUCKET_NAME: process.env.GCS_BUCKET_NAME
      },
      watch: false,
      instances: 1,
      autorestart: true,
      max_memory_restart: '1G'
    },

    // LLM Service - AI Processing (Port 8080)
    {
      name: 'llm-service',
      cwd: './llm-service/api',
      script: 'uvicorn',
      args: 'main:app --reload --host 0.0.0.0 --port 8080',
      interpreter: 'python3',
      env: {
        OPENAI_API_KEY: process.env.OPENAI_API_KEY,
        ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
        GOOGLE_CLOUD_PROJECT: process.env.GOOGLE_CLOUD_PROJECT,
        VERTEX_LOCATION: process.env.VERTEX_LOCATION || 'us-central1'
      },
      watch: false,
      instances: 1,
      autorestart: true,
      max_memory_restart: '2G'
    },

    // Router Service - Job Routing (Port 9001)
    {
      name: 'router',
      cwd: './router',
      script: 'uvicorn',
      args: 'main:app --reload --host 0.0.0.0 --port 9001',
      interpreter: 'python3',
      env: {
        REDIS_HOST: 'localhost',
        REDIS_PORT: 6379,
        DMS_URL: 'http://localhost:1337',
        ROUTER_API_KEY: process.env.ROUTER_API_KEY || '6f2452d7-c1c1-422e-9cb6-e958d560e06b'
      },
      watch: false,
      instances: 1,
      autorestart: true,
      max_memory_restart: '512M'
    },

    // RAG Query Service - Query Engine (Port 8003)
    {
      name: 'rag-query',
      cwd: './rag_query_service',
      script: 'uvicorn',
      args: 'main:app --reload --host 0.0.0.0 --port 8003',
      interpreter: 'python3',
      env: {
        POSTGRES_HOST: 'localhost',
        POSTGRES_PORT: 5433,
        POSTGRES_DB: 'turfai_rag',
        POSTGRES_USER: 'postgres',
        POSTGRES_PASSWORD: 'postgres',
        DMS_URL: 'http://localhost:1337',
        LLM_SERVICE_URL: 'http://localhost:8080'
      },
      watch: false,
      instances: 1,
      autorestart: true,
      max_memory_restart: '1G'
    },

    // Processor - Document Processing Worker
    {
      name: 'processor',
      cwd: './processors',
      script: 'main.py',
      interpreter: 'python3',
      env: {
        REDIS_HOST: 'localhost',
        REDIS_PORT: 6379,
        DMS_URL: 'http://localhost:1337',
        LLM_SERVICE_URL: 'http://localhost:8080',
        RAG_PROCESSOR_URL: 'http://localhost:8002'
      },
      watch: false,
      instances: 1,
      autorestart: true,
      max_memory_restart: '1G'
    }
  ]
};

#!/bin/bash
set -e

echo "🚀 Starting TurfAI Local Development Environment..."
echo ""

# Colors for output
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m' # No Color

# Check if .env.local exists
if [ ! -f .env.local ]; then
    echo -e "${RED}❌ .env.local not found!${NC}"
    echo "Please copy .env.example to .env.local and configure it"
    exit 1
fi

# Load environment variables
export $(grep -v '^#' .env.local | xargs)

# Check dependencies
echo -e "${YELLOW}📋 Checking dependencies...${NC}"

# Check PostgreSQL
if ! command -v psql &> /dev/null; then
    echo -e "${RED}❌ PostgreSQL not found. Please install: brew install postgresql@15${NC}"
    exit 1
fi

# Check Redis
if ! command -v redis-cli &> /dev/null; then
    echo -e "${RED}❌ Redis not found. Please install: brew install redis${NC}"
    exit 1
fi

# Check PM2
if ! command -v pm2 &> /dev/null; then
    echo -e "${RED}❌ PM2 not found. Installing...${NC}"
    npm install -g pm2
fi

# Check Python
if ! command -v python3 &> /dev/null; then
    echo -e "${RED}❌ Python3 not found. Please install Python 3.8+${NC}"
    exit 1
fi

# Check Node.js
if ! command -v node &> /dev/null; then
    echo -e "${RED}❌ Node.js not found. Please install Node.js 16+${NC}"
    exit 1
fi

echo -e "${GREEN}✅ All dependencies found${NC}"
echo ""

# Start PostgreSQL if not running
echo -e "${YELLOW}🐘 Starting PostgreSQL...${NC}"
if ! pg_isready -q; then
    brew services start postgresql@15
    sleep 2
fi
echo -e "${GREEN}✅ PostgreSQL running${NC}"

# Start Redis if not running
echo -e "${YELLOW}📦 Starting Redis...${NC}"
if ! redis-cli ping &> /dev/null; then
    brew services start redis
    sleep 2
fi
echo -e "${GREEN}✅ Redis running${NC}"

# Initialize databases (first time only)
if [ ! -f .db-initialized ]; then
    echo -e "${YELLOW}🗄️  Initializing databases...${NC}"
    ./dev/scripts/init-db.sh
    touch .db-initialized
    echo -e "${GREEN}✅ Databases initialized${NC}"
else
    echo -e "${GREEN}✅ Databases already initialized${NC}"
fi

echo ""
echo -e "${YELLOW}🚀 Starting all services with PM2...${NC}"

# Start all services
pm2 start dev/ecosystem.config.js

echo ""
echo -e "${GREEN}✅ All services started successfully!${NC}"
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
echo "📊 Service Status:"
echo "  • DMS:              http://localhost:1337"
echo "  • LLM Service:      http://localhost:8080"
echo "  • Router:           http://localhost:9001"
echo "  • RAG Query:        http://localhost:8003"
echo "  • Processor:        Running in background"
echo ""
echo "📋 Useful Commands:"
echo "  • View logs:        pm2 logs"
echo "  • View logs (one):  pm2 logs dms"
echo "  • Monitor:          pm2 monit"
echo "  • Status:           pm2 status"
echo "  • Stop all:         ./dev/scripts/stop-local.sh"
echo "  • Restart one:      pm2 restart dms"
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

chmod +x dev/scripts/start-local.sh

#!/bin/bash
set -e

echo "🛑 Stopping TurfAI services..."

pm2 delete all

echo "✅ All services stopped"
echo ""
echo "Note: PostgreSQL and Redis are still running"
echo "To stop them:"
echo "  brew services stop postgresql@15"
echo "  brew services stop redis"

chmod +x dev/scripts/stop-local.sh

#!/bin/bash

# View logs for specific service or all
if [ -z "$1" ]; then
    echo "📊 Viewing logs for ALL services (Ctrl+C to exit)"
    pm2 logs
else
    echo "📊 Viewing logs for: $1"
    pm2 logs "$1"
fi

chmod +x dev/scripts/logs.sh

#!/bin/bash
set -e

echo "📦 Initializing TurfAI Databases..."
echo ""

# Colors
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# 1. Create main database for DMS
echo -e "${YELLOW}Creating turfai_dms database...${NC}"
psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'turfai_dms'" | grep -q 1 || psql -U postgres -c "CREATE DATABASE turfai_dms"
echo -e "${GREEN}✅ turfai_dms database ready${NC}"

# 2. Create RAG database with pgvector
echo -e "${YELLOW}Creating turfai_rag database with pgvector...${NC}"
psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'turfai_rag'" | grep -q 1 || psql -U postgres -c "CREATE DATABASE turfai_rag"
psql -U postgres -d turfai_rag -c "CREATE EXTENSION IF NOT EXISTS vector"
echo -e "${GREEN}✅ turfai_rag database ready with pgvector${NC}"

# 3. Run RAG schema
echo -e "${YELLOW}Creating RAG tables...${NC}"
if [ -f rag_query_service/schema.sql ]; then
    psql -U postgres -d turfai_rag -f rag_query_service/schema.sql
    echo -e "${GREEN}✅ RAG tables created${NC}"
else
    echo -e "${YELLOW}⚠️  RAG schema.sql not found, skipping...${NC}"
fi

# 4. DMS will auto-migrate on first start
echo -e "${GREEN}✅ DMS will auto-migrate on first start${NC}"

echo ""
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${GREEN}✅ Database initialization complete!${NC}"
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo ""
echo "Next steps:"
echo "1. Start services: ./dev/scripts/start-local.sh"
echo "2. Create Strapi admin user: http://localhost:1337/admin"

chmod +x dev/scripts/init-db.sh

const { v4: uuidv4 } = require('uuid');

module.exports = (config, { strapi }) => {
  return async (ctx, next) => {
    // Get correlation ID from header or generate new one
    const correlationId = ctx.request.header['x-correlation-id'] || uuidv4();

    // Store in context
    ctx.correlationId = correlationId;

    // Add to response headers
    ctx.set('x-correlation-id', correlationId);

    // Add to Strapi logger
    strapi.log.info(`[${correlationId}] ${ctx.method} ${ctx.url}`);

    await next();
  };
};

module.exports = [
  'strapi::logger',
  'strapi::errors',
  'strapi::security',
  'global::correlation-id',  // ADD THIS
  'strapi::cors',
  // ... rest
];

// Example: dms/src/api/rag/controllers/rag.js
async query(ctx) {
  const correlationId = ctx.correlationId;

  const response = await axios.post(
    `${ragQueryUrl}/api/v1/rag/query`,
    ragRequest,
    {
      headers: {
        'Authorization': authHeader,
        'Content-Type': 'application/json',
        'x-correlation-id': correlationId  // PASS IT ALONG
      }
    }
  );

  strapi.log.info(`[${correlationId}] RAG query completed`);
  return ctx.send(response.data);
}

import logging
from contextvars import ContextVar
from typing import Optional

# Context variable to store correlation ID
correlation_id_var: ContextVar[Optional[str]] = ContextVar('correlation_id', default=None)

class CorrelationFilter(logging.Filter):
    """Add correlation ID to all log records."""

    def filter(self, record):
        correlation_id = correlation_id_var.get()
        record.correlation_id = correlation_id or 'NO_CORRELATION_ID'
        return True

def setup_correlation_logging():
    """Setup logging with correlation ID support."""

    # Create formatter with correlation ID
    formatter = logging.Formatter(
        '%(asctime)s - [%(correlation_id)s] - %(name)s - %(levelname)s - %(message)s'
    )

    # Get root logger
    logger = logging.getLogger()

    # Add correlation filter to all handlers
    for handler in logger.handlers:
        handler.addFilter(CorrelationFilter())
        handler.setFormatter(formatter)

def set_correlation_id(correlation_id: str):
    """Set correlation ID for current context."""
    correlation_id_var.set(correlation_id)

def get_correlation_id() -> Optional[str]:
    """Get correlation ID from current context."""
    return correlation_id_var.get()

from fastapi import FastAPI, Request
from processors.utils.correlation_logger import (
    setup_correlation_logging,
    set_correlation_id,
    get_correlation_id
)
import uuid
import logging

app = FastAPI()

# Setup correlation logging
setup_correlation_logging()
logger = logging.getLogger(__name__)

@app.middleware("http")
async def correlation_id_middleware(request: Request, call_next):
    # Extract correlation ID from header or generate new one
    correlation_id = request.headers.get('x-correlation-id', str(uuid.uuid4()))

    # Set in context
    set_correlation_id(correlation_id)

    # Log incoming request
    logger.info(f"Incoming request: {request.method} {request.url.path}")

    # Process request
    response = await call_next(request)

    # Add correlation ID to response headers
    response.headers['x-correlation-id'] = correlation_id

    return response

User Request → DMS
[abc-123] DMS: POST /api/rag/query
    ↓
[abc-123] DMS: Forwarding to RAG Query Service
    ↓
[abc-123] RAG Query: Received query request
    ↓
[abc-123] RAG Query: Calling LLM Service
    ↓
[abc-123] LLM Service: Processing chat request
    ↓
[abc-123] LLM Service: Response generated
    ↓
[abc-123] RAG Query: Query completed
    ↓
[abc-123] DMS: Returning response to client

# View all logs
pm2 logs

# Search for specific correlation ID
pm2 logs | grep "abc-123"

# Or use jq for JSON logs
pm2 logs --json | jq 'select(.correlation_id == "abc-123")'

brew install lnav

# View PM2 logs with filtering
pm2 logs --raw > /tmp/turfai.log &
lnav /tmp/turfai.log

# In lnav, press '/' to search for correlation ID

version: '3.8'

services:
  # PostgreSQL for DMS
  postgres:
    image: postgres:15
    container_name: turfai-postgres
    environment:
      POSTGRES_DB: turfai_dms
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: postgres
    volumes:
      - postgres_data:/var/lib/postgresql/data
    ports:
      - "5432:5432"
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U postgres"]
      interval: 5s
      timeout: 5s
      retries: 5
    networks:
      - turfai-network

  # PostgreSQL with pgvector for RAG
  postgres-rag:
    image: pgvector/pgvector:pg15
    container_name: turfai-postgres-rag
    environment:
      POSTGRES_DB: turfai_rag
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: postgres
    volumes:
      - postgres_rag_data:/var/lib/postgresql/data
      - ./rag_query_service/schema.sql:/docker-entrypoint-initdb.d/schema.sql
    ports:
      - "5433:5432"
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U postgres"]
      interval: 5s
      timeout: 5s
      retries: 5
    networks:
      - turfai-network

  # Redis for job queue
  redis:
    image: redis:7-alpine
    container_name: turfai-redis
    ports:
      - "6379:6379"
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      timeout: 3s
      retries: 5
    networks:
      - turfai-network

  # DMS - Strapi CMS
  dms:
    build:
      context: ./dms
      dockerfile: Dockerfile
    container_name: turfai-dms
    ports:
      - "1337:1337"
    environment:
      DATABASE_HOST: postgres
      DATABASE_PORT: 5432
      DATABASE_NAME: turfai_dms
      DATABASE_USERNAME: postgres
      DATABASE_PASSWORD: postgres
      DATABASE_SSL: "false"
      GCS_BUCKET_NAME: ${GCS_BUCKET_NAME}
      GOOGLE_APPLICATION_CREDENTIALS: /app/gcp-credentials.json
    depends_on:
      postgres:
        condition: service_healthy
    volumes:
      - ./dms:/app
      - /app/node_modules
      - ${GOOGLE_APPLICATION_CREDENTIALS}:/app/gcp-credentials.json:ro
    networks:
      - turfai-network
    labels:
      - "service=dms"

  # LLM Service
  llm-service:
    build:
      context: ./llm-service
      dockerfile: api/Dockerfile
    container_name: turfai-llm-service
    ports:
      - "8080:8080"
    environment:
      OPENAI_API_KEY: ${OPENAI_API_KEY}
      ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY}
      GOOGLE_CLOUD_PROJECT: ${GOOGLE_CLOUD_PROJECT}
      VERTEX_LOCATION: ${VERTEX_LOCATION:-us-central1}
      GOOGLE_APPLICATION_CREDENTIALS: /app/gcp-credentials.json
    volumes:
      - ./llm-service:/app
      - ${GOOGLE_APPLICATION_CREDENTIALS}:/app/gcp-credentials.json:ro
    networks:
      - turfai-network
    labels:
      - "service=llm-service"

  # Router Service
  router:
    build:
      context: ./router
      dockerfile: Dockerfile
    container_name: turfai-router
    ports:
      - "9001:9001"
    environment:
      REDIS_HOST: redis
      REDIS_PORT: 6379
      DMS_URL: http://dms:1337
      ROUTER_API_KEY: ${ROUTER_API_KEY:-6f2452d7-c1c1-422e-9cb6-e958d560e06b}
    depends_on:
      redis:
        condition: service_healthy
      dms:
        condition: service_started
    volumes:
      - ./router:/app
    networks:
      - turfai-network
    labels:
      - "service=router"

  # RAG Query Service
  rag-query:
    build:
      context: ./rag_query_service
      dockerfile: Dockerfile
    container_name: turfai-rag-query
    ports:
      - "8003:8003"
    environment:
      POSTGRES_HOST: postgres-rag
      POSTGRES_PORT: 5432
      POSTGRES_DB: turfai_rag
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: postgres
      DMS_URL: http://dms:1337
      LLM_SERVICE_URL: http://llm-service:8080
    depends_on:
      postgres-rag:
        condition: service_healthy
      llm-service:
        condition: service_started
    volumes:
      - ./rag_query_service:/app
    networks:
      - turfai-network
    labels:
      - "service=rag-query"

  # Processor Service
  processor:
    build:
      context: ./processors
      dockerfile: Dockerfile
    container_name: turfai-processor
    environment:
      REDIS_HOST: redis
      REDIS_PORT: 6379
      DMS_URL: http://dms:1337
      LLM_SERVICE_URL: http://llm-service:8080
      GOOGLE_APPLICATION_CREDENTIALS: /app/gcp-credentials.json
    depends_on:
      redis:
        condition: service_healthy
      dms:
        condition: service_started
    volumes:
      - ./processors:/app
      - ${GOOGLE_APPLICATION_CREDENTIALS}:/app/gcp-credentials.json:ro
    networks:
      - turfai-network
    labels:
      - "service=processor"

volumes:
  postgres_data:
  postgres_rag_data:

networks:
  turfai-network:
    driver: bridge

# Copy example
cp .env.example .env

# Edit with your values
GOOGLE_CLOUD_PROJECT=your-project
VERTEX_LOCATION=us-central1
GCS_BUCKET_NAME=your-bucket
OPENAI_API_KEY=sk-...
ANTHROPIC_API_KEY=sk-ant-...
GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json
ROUTER_API_KEY=6f2452d7-c1c1-422e-9cb6-e958d560e06b

# Start all services
docker-compose up -d

# View logs (all services)
docker-compose logs -f

# View logs (specific service)
docker-compose logs -f dms

# View logs with correlation ID
docker-compose logs -f | grep "abc-123"

# Check service status
docker-compose ps

# Restart specific service
docker-compose restart dms

# Stop all services
docker-compose down

# Stop and remove volumes (fresh start)
docker-compose down -v

# Rebuild after code changes
docker-compose up -d --build dms

Internet
    ↓
Frontend (Vercel/Netlify)
    ↓ HTTPS
Cloud Load Balancer
    ↓
DMS (Cloud Run - Public)
    ↓
Internal Services (Cloud Run - Private Ingress):
    ├─ LLM Service
    ├─ Router
    ├─ RAG Query Service
    └─ Processor (Cloud Run Jobs)

Managed Services:
    ├─ Cloud SQL (PostgreSQL + pgvector)
    ├─ Memorystore (Redis)
    └─ Cloud Storage (GCS)

# Install Google Cloud SDK
brew install --cask google-cloud-sdk

# Authenticate
gcloud auth login

# Set project
gcloud config set project YOUR_PROJECT_ID

# Enable APIs
gcloud services enable \
  run.googleapis.com \
  sqladmin.googleapis.com \
  redis.googleapis.com \
  cloudbuild.googleapis.com \
  secretmanager.googleapis.com

# Create PostgreSQL instance for DMS
gcloud sql instances create turfai-postgres \
  --database-version=POSTGRES_15 \
  --tier=db-g1-small \
  --region=us-central1 \
  --root-password=CHANGE_ME

# Create database
gcloud sql databases create turfai_dms \
  --instance=turfai-postgres

# Create PostgreSQL instance for RAG (with more resources)
gcloud sql instances create turfai-postgres-rag \
  --database-version=POSTGRES_15 \
  --tier=db-custom-2-8192 \
  --region=us-central1 \
  --root-password=CHANGE_ME

# Create database
gcloud sql databases create turfai_rag \
  --instance=turfai-postgres-rag

# Install pgvector extension (manual step via Cloud SQL proxy)
gcloud sql connect turfai-postgres-rag --user=postgres
# Then: CREATE EXTENSION vector;

gcloud redis instances create turfai-redis \
  --size=1 \
  --region=us-central1 \
  --redis-version=redis_7_0

# Store API keys in Secret Manager
echo -n "sk-your-openai-key" | gcloud secrets create openai-api-key --data-file=-
echo -n "sk-ant-your-anthropic-key" | gcloud secrets create anthropic-api-key --data-file=-
echo -n "your-router-api-key" | gcloud secrets create router-api-key --data-file=-

# GCP service account key (if needed)
gcloud secrets create gcp-credentials --data-file=/path/to/credentials.json

#!/bin/bash
set -e

# Configuration
PROJECT_ID="your-gcp-project"
REGION="us-central1"
DMS_SQL_INSTANCE="turfai-postgres"
RAG_SQL_INSTANCE="turfai-postgres-rag"
REDIS_HOST="10.0.0.3"  # Get from: gcloud redis instances describe turfai-redis

echo "🚀 Deploying TurfAI to Cloud Run..."
echo "Project: $PROJECT_ID"
echo "Region: $REGION"
echo ""

# 1. Deploy DMS (Public Ingress - API Gateway)
echo "📦 Deploying DMS..."
gcloud run deploy turfai-dms \
  --source ./dms \
  --region $REGION \
  --allow-unauthenticated \
  --platform managed \
  --set-env-vars DATABASE_HOST=/cloudsql/$PROJECT_ID:$REGION:$DMS_SQL_INSTANCE,DATABASE_NAME=turfai_dms,DATABASE_USERNAME=postgres \
  --set-secrets DATABASE_PASSWORD=db-password:latest,GCS_BUCKET_NAME=gcs-bucket:latest \
  --add-cloudsql-instances $PROJECT_ID:$REGION:$DMS_SQL_INSTANCE \
  --memory 1Gi \
  --cpu 1 \
  --min-instances 1 \
  --max-instances 10 \
  --timeout 300 \
  --concurrency 80

DMS_URL=$(gcloud run services describe turfai-dms --region $REGION --format 'value(status.url)')
echo "✅ DMS deployed: $DMS_URL"

# 2. Deploy LLM Service (Private Ingress)
echo "📦 Deploying LLM Service..."
gcloud run deploy turfai-llm \
  --source ./llm-service/api \
  --region $REGION \
  --no-allow-unauthenticated \
  --platform managed \
  --set-secrets OPENAI_API_KEY=openai-api-key:latest,ANTHROPIC_API_KEY=anthropic-api-key:latest,GOOGLE_APPLICATION_CREDENTIALS=gcp-credentials:latest \
  --set-env-vars VERTEX_LOCATION=$REGION \
  --memory 2Gi \
  --cpu 2 \
  --min-instances 0 \
  --max-instances 5 \
  --timeout 300

LLM_URL=$(gcloud run services describe turfai-llm --region $REGION --format 'value(status.url)')
echo "✅ LLM Service deployed: $LLM_URL"

# 3. Deploy Router (Private Ingress)
echo "📦 Deploying Router..."
gcloud run deploy turfai-router \
  --source ./router \
  --region $REGION \
  --no-allow-unauthenticated \
  --platform managed \
  --set-env-vars REDIS_HOST=$REDIS_HOST,REDIS_PORT=6379,DMS_URL=$DMS_URL \
  --set-secrets ROUTER_API_KEY=router-api-key:latest \
  --memory 512Mi \
  --cpu 1 \
  --min-instances 0 \
  --max-instances 10 \
  --timeout 60

ROUTER_URL=$(gcloud run services describe turfai-router --region $REGION --format 'value(status.url)')
echo "✅ Router deployed: $ROUTER_URL"

# 4. Deploy RAG Query Service (Private Ingress)
echo "📦 Deploying RAG Query Service..."
gcloud run deploy turfai-rag-query \
  --source ./rag_query_service \
  --region $REGION \
  --no-allow-unauthenticated \
  --platform managed \
  --set-env-vars POSTGRES_HOST=/cloudsql/$PROJECT_ID:$REGION:$RAG_SQL_INSTANCE,POSTGRES_DB=turfai_rag,POSTGRES_USER=postgres,DMS_URL=$DMS_URL,LLM_SERVICE_URL=$LLM_URL \
  --set-secrets POSTGRES_PASSWORD=db-password:latest \
  --add-cloudsql-instances $PROJECT_ID:$REGION:$RAG_SQL_INSTANCE \
  --memory 1Gi \
  --cpu 1 \
  --min-instances 0 \
  --max-instances 5 \
  --timeout 300

RAG_QUERY_URL=$(gcloud run services describe turfai-rag-query --region $REGION --format 'value(status.url)')
echo "✅ RAG Query deployed: $RAG_QUERY_URL"

# 5. Deploy Processor as Cloud Run Job
echo "📦 Deploying Processor (Cloud Run Job)..."
gcloud run jobs create turfai-processor \
  --source ./processors \
  --region $REGION \
  --set-env-vars REDIS_HOST=$REDIS_HOST,REDIS_PORT=6379,DMS_URL=$DMS_URL,LLM_SERVICE_URL=$LLM_URL \
  --set-secrets GOOGLE_APPLICATION_CREDENTIALS=gcp-credentials:latest \
  --memory 1Gi \
  --cpu 1 \
  --task-timeout 1h \
  --max-retries 3

echo "✅ Processor job created"

# Update DMS with internal service URLs
echo "📝 Updating DMS environment variables..."
gcloud run services update turfai-dms \
  --region $REGION \
  --set-env-vars LLM_SERVICE_URL=$LLM_URL,ROUTER_URL=$ROUTER_URL,RAG_QUERY_SERVICE_URL=$RAG_QUERY_URL

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "✅ Deployment Complete!"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
echo "Public Endpoint:"
echo "  DMS: $DMS_URL"
echo ""
echo "Internal Services (accessible only from DMS):"
echo "  LLM Service:   $LLM_URL"
echo "  Router:        $ROUTER_URL"
echo "  RAG Query:     $RAG_QUERY_URL"
echo ""
echo "Next Steps:"
echo "1. Update frontend to use: $DMS_URL"
echo "2. Run end-to-end tests"
echo "3. View logs: gcloud logging tail"

chmod +x dev/scripts/deploy-cloud-run.sh

# View all logs
gcloud logging tail

# View logs for specific service
gcloud logging tail --filter='resource.labels.service_name="turfai-dms"'

# Search by correlation ID
gcloud logging tail --filter='jsonPayload.correlation_id="abc-123"'

# View logs from last hour
gcloud logging tail --since=1h

# Export logs to BigQuery for analysis
gcloud logging sinks create turfai-logs \
  bigquery.googleapis.com/projects/YOUR_PROJECT/datasets/turfai_logs \
  --log-filter='resource.type="cloud_run_revision"'

# Test DMS
curl http://localhost:1337/_health

# Test LLM Service
curl http://localhost:8080/health

# Test Router
curl http://localhost:9001/health

# Test RAG Query
curl http://localhost:8003/health

# 1. Upload document via DMS
curl -X POST http://localhost:1337/api/upload \
  -H "Authorization: Bearer YOUR_TOKEN" \
  -F "files=@test-document.pdf"

# 2. Check document in DMS
curl http://localhost:1337/api/documents/1 \
  -H "Authorization: Bearer YOUR_TOKEN"

# 3. Enable RAG for document
curl -X POST http://localhost:1337/api/documents/1/enable-rag \
  -H "Authorization: Bearer YOUR_TOKEN"

# 4. Check processing status
curl http://localhost:1337/api/documents/1/rag-status \
  -H "Authorization: Bearer YOUR_TOKEN"

# Query via DMS (which proxies to RAG Query Service)
curl -X POST http://localhost:1337/api/rag/query \
  -H "Authorization: Bearer YOUR_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{
    "query": "What is the main topic of the document?",
    "session_id": null,
    "top_k": 5
  }'

# PM2
pm2 logs | grep "abc-123"

# Docker Compose
docker-compose logs -f | grep "abc-123"

# Cloud Run
gcloud logging tail --filter='jsonPayload.correlation_id="abc-123"'

# Test extraction via DMS
curl -X POST http://localhost:1337/api/llm/extract \
  -H "Authorization: Bearer YOUR_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{
    "text": "Invoice #12345\nDate: 2025-01-01\nTotal: $1,234.56",
    "schema": {
      "type": "object",
      "properties": {
        "invoice_number": {"type": "string"},
        "date": {"type": "string"},
        "total": {"type": "number"}
      }
    }
  }'

# Create session
curl -X POST http://localhost:1337/api/rag/sessions \
  -H "Authorization: Bearer YOUR_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{"title": "Test Session"}'
# Returns: {"session_id": "uuid-here"}

# First query
curl -X POST http://localhost:1337/api/rag/query \
  -H "Authorization: Bearer YOUR_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{
    "query": "What is the name on the Aadhar card?",
    "session_id": "uuid-here"
  }'

# Follow-up query (should use context)
curl -X POST http://localhost:1337/api/rag/query \
  -H "Authorization: Bearer YOUR_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{
    "query": "What is the address?",
    "session_id": "uuid-here"
  }'

# Start everything
./dev/scripts/start-local.sh

# View logs
pm2 logs

# View logs for one service
pm2 logs dms

# Restart one service
pm2 restart dms

# Stop everything
./dev/scripts/stop-local.sh

# Start
docker-compose up -d

# Logs
docker-compose logs -f

# Logs for one service
docker-compose logs -f dms

# Restart one service
docker-compose restart dms

# Stop
docker-compose down

# Deploy
./dev/scripts/deploy-cloud-run.sh

# View logs
gcloud logging tail

# View logs for one service
gcloud logging tail --filter='resource.labels.service_name="turfai-dms"'

# Search by correlation ID
gcloud logging tail --filter='jsonPayload.correlation_id="abc-123"'

# Check logs
pm2 logs <service-name>

# Check if port is in use
lsof -i :<port>

# Kill process on port
kill -9 <PID>

# Check PostgreSQL is running
pg_isready

# Start PostgreSQL
brew services start postgresql@15

# Check connection
psql -U postgres -l

# Check Redis is running
redis-cli ping

# Start Redis
brew services start redis