feat(observability): add mothership tracing

apps/sim/app/api/billing/update-cost/route.ts

@@ -5,7 +5,11 @@ import { type NextRequest, NextResponse } from 'next/server'
 import { z } from 'zod'
 import { recordUsage } from '@/lib/billing/core/usage-log'
 import { checkAndBillOverageThreshold } from '@/lib/billing/threshold-billing'
+import { BillingRouteOutcome } from '@/lib/copilot/generated/trace-attribute-values-v1'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1'
 import { checkInternalApiKey } from '@/lib/copilot/request/http'
+import { withIncomingGoSpan } from '@/lib/copilot/request/otel'
 import { isBillingEnabled } from '@/lib/core/config/feature-flags'
 import { type AtomicClaimResult, billingIdempotency } from '@/lib/core/idempotency/service'
 import { generateRequestId } from '@/lib/core/utils/request'
@@ -28,8 +32,28 @@ const UpdateCostSchema = z.object({
 /**
  * POST /api/billing/update-cost
  * Update user cost with a pre-calculated cost value (internal API key auth required)
+ *
+ * Parented under the Go-side `sim.update_cost` span via W3C traceparent
+ * propagation. Every mothership request that bills should therefore show
+ * the Go client span AND this Sim server span sharing one trace, with
+ * the actual usage/overage work nested below.
  */
-export const POST = withRouteHandler(async (req: NextRequest) => {
+export const POST = withRouteHandler((req: NextRequest) =>
+  withIncomingGoSpan(
+    req.headers,
+    TraceSpan.CopilotBillingUpdateCost,
+    {
+      [TraceAttr.HttpMethod]: 'POST',
+      [TraceAttr.HttpRoute]: '/api/billing/update-cost',
+    },
+    async (span) => updateCostInner(req, span)
+  )
+)
+
+async function updateCostInner(
+  req: NextRequest,
+  span: import('@opentelemetry/api').Span
+): Promise<NextResponse> {
   const requestId = generateRequestId()
   const startTime = Date.now()
   let claim: AtomicClaimResult | null = null
@@ -39,6 +63,8 @@ export const POST = withRouteHandler(async (req: NextRequest) => {
     logger.info(`[${requestId}] Update cost request started`)
 
     if (!isBillingEnabled) {
+      span.setAttribute(TraceAttr.BillingOutcome, BillingRouteOutcome.BillingDisabled)
+      span.setAttribute(TraceAttr.HttpStatusCode, 200)
       return NextResponse.json({
         success: true,
         message: 'Billing disabled, cost update skipped',
@@ -54,6 +80,8 @@ export const POST = withRouteHandler(async (req: NextRequest) => {
     const authResult = checkInternalApiKey(req)
     if (!authResult.success) {
       logger.warn(`[${requestId}] Authentication failed: ${authResult.error}`)
+      span.setAttribute(TraceAttr.BillingOutcome, BillingRouteOutcome.AuthFailed)
+      span.setAttribute(TraceAttr.HttpStatusCode, 401)
       return NextResponse.json(
         {
           success: false,
@@ -69,8 +97,9 @@ export const POST = withRouteHandler(async (req: NextRequest) => {
     if (!validation.success) {
       logger.warn(`[${requestId}] Invalid request body`, {
         errors: validation.error.issues,
-        body,
       })
+      span.setAttribute(TraceAttr.BillingOutcome, BillingRouteOutcome.InvalidBody)
+      span.setAttribute(TraceAttr.HttpStatusCode, 400)
       return NextResponse.json(
         {
           success: false,
@@ -85,6 +114,17 @@ export const POST = withRouteHandler(async (req: NextRequest) => {
       validation.data
     const isMcp = source === 'mcp_copilot'
 
+    span.setAttributes({
+      [TraceAttr.UserId]: userId,
+      [TraceAttr.GenAiRequestModel]: model,
+      [TraceAttr.BillingSource]: source,
+      [TraceAttr.BillingCostUsd]: cost,
+      [TraceAttr.GenAiUsageInputTokens]: inputTokens,
+      [TraceAttr.GenAiUsageOutputTokens]: outputTokens,
+      [TraceAttr.BillingIsMcp]: isMcp,
+      ...(idempotencyKey ? { [TraceAttr.BillingIdempotencyKey]: idempotencyKey } : {}),
+    })
+
     claim = idempotencyKey
       ? await billingIdempotency.atomicallyClaim('update-cost', idempotencyKey)
       : null
@@ -95,6 +135,8 @@ export const POST = withRouteHandler(async (req: NextRequest) => {
         userId,
         source,
       })
+      span.setAttribute(TraceAttr.BillingOutcome, BillingRouteOutcome.DuplicateIdempotencyKey)
+      span.setAttribute(TraceAttr.HttpStatusCode, 409)
       return NextResponse.json(
         {
           success: false,
@@ -159,6 +201,9 @@ export const POST = withRouteHandler(async (req: NextRequest) => {
       cost,
     })
 
+    span.setAttribute(TraceAttr.BillingOutcome, BillingRouteOutcome.Billed)
+    span.setAttribute(TraceAttr.HttpStatusCode, 200)
+    span.setAttribute(TraceAttr.BillingDurationMs, duration)
     return NextResponse.json({
       success: true,
       data: {
@@ -193,6 +238,9 @@ export const POST = withRouteHandler(async (req: NextRequest) => {
       )
     }
 
+    span.setAttribute(TraceAttr.BillingOutcome, BillingRouteOutcome.InternalError)
+    span.setAttribute(TraceAttr.HttpStatusCode, 500)
+    span.setAttribute(TraceAttr.BillingDurationMs, duration)
     return NextResponse.json(
       {
         success: false,
@@ -202,4 +250,4 @@ export const POST = withRouteHandler(async (req: NextRequest) => {
       { status: 500 }
     )
   }
-})
+}

apps/sim/app/api/copilot/api-keys/generate/route.ts

@@ -2,6 +2,8 @@ import { type NextRequest, NextResponse } from 'next/server'
 import { z } from 'zod'
 import { getSession } from '@/lib/auth'
 import { SIM_AGENT_API_URL } from '@/lib/copilot/constants'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { fetchGo } from '@/lib/copilot/request/go/fetch'
 import { env } from '@/lib/core/config/env'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 
@@ -33,13 +35,16 @@ export const POST = withRouteHandler(async (req: NextRequest) => {
 
     const { name } = validationResult.data
 
-    const res = await fetch(`${SIM_AGENT_API_URL}/api/validate-key/generate`, {
+    const res = await fetchGo(`${SIM_AGENT_API_URL}/api/validate-key/generate`, {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
         ...(env.COPILOT_API_KEY ? { 'x-api-key': env.COPILOT_API_KEY } : {}),
       },
       body: JSON.stringify({ userId, name }),
+      spanName: 'sim → go /api/validate-key/generate',
+      operation: 'generate_api_key',
+      attributes: { [TraceAttr.UserId]: userId },
     })
 
     if (!res.ok) {

apps/sim/app/api/copilot/api-keys/route.ts

@@ -1,6 +1,8 @@
 import { type NextRequest, NextResponse } from 'next/server'
 import { getSession } from '@/lib/auth'
 import { SIM_AGENT_API_URL } from '@/lib/copilot/constants'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { fetchGo } from '@/lib/copilot/request/go/fetch'
 import { env } from '@/lib/core/config/env'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 
@@ -13,13 +15,16 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
 
     const userId = session.user.id
 
-    const res = await fetch(`${SIM_AGENT_API_URL}/api/validate-key/get-api-keys`, {
+    const res = await fetchGo(`${SIM_AGENT_API_URL}/api/validate-key/get-api-keys`, {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
         ...(env.COPILOT_API_KEY ? { 'x-api-key': env.COPILOT_API_KEY } : {}),
       },
       body: JSON.stringify({ userId }),
+      spanName: 'sim → go /api/validate-key/get-api-keys',
+      operation: 'get_api_keys',
+      attributes: { [TraceAttr.UserId]: userId },
     })
 
     if (!res.ok) {
@@ -67,13 +72,16 @@ export const DELETE = withRouteHandler(async (request: NextRequest) => {
       return NextResponse.json({ error: 'id is required' }, { status: 400 })
     }
 
-    const res = await fetch(`${SIM_AGENT_API_URL}/api/validate-key/delete`, {
+    const res = await fetchGo(`${SIM_AGENT_API_URL}/api/validate-key/delete`, {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
         ...(env.COPILOT_API_KEY ? { 'x-api-key': env.COPILOT_API_KEY } : {}),
       },
       body: JSON.stringify({ userId, apiKeyId: id }),
+      spanName: 'sim → go /api/validate-key/delete',
+      operation: 'delete_api_key',
+      attributes: { [TraceAttr.UserId]: userId, [TraceAttr.ApiKeyId]: id },
     })
 
     if (!res.ok) {

apps/sim/app/api/copilot/api-keys/validate/route.ts

@@ -5,7 +5,11 @@ import { eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { z } from 'zod'
 import { checkServerSideUsageLimits } from '@/lib/billing/calculations/usage-monitor'
+import { CopilotValidateOutcome } from '@/lib/copilot/generated/trace-attribute-values-v1'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1'
 import { checkInternalApiKey } from '@/lib/copilot/request/http'
+import { withIncomingGoSpan } from '@/lib/copilot/request/otel'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 
 const logger = createLogger('CopilotApiKeysValidate')
@@ -14,55 +18,87 @@ const ValidateApiKeySchema = z.object({
   userId: z.string().min(1, 'userId is required'),
 })
 
-export const POST = withRouteHandler(async (req: NextRequest) => {
-  try {
-    const auth = checkInternalApiKey(req)
-    if (!auth.success) {
-      return new NextResponse(null, { status: 401 })
-    }
-
-    const body = await req.json().catch(() => null)
+// Incoming-from-Go: extracts traceparent so this handler's work shows
+// up as a child of the Go-side `sim.validate_api_key` span in the same
+// trace. If there's no traceparent (manual curl / browser), the helper
+// falls back to a new root span.
+export const POST = withRouteHandler((req: NextRequest) =>
+  withIncomingGoSpan(
+    req.headers,
+    TraceSpan.CopilotAuthValidateApiKey,
+    {
+      [TraceAttr.HttpMethod]: 'POST',
+      [TraceAttr.HttpRoute]: '/api/copilot/api-keys/validate',
+    },
+    async (span) => {
+      try {
+        const auth = checkInternalApiKey(req)
+        if (!auth.success) {
+          span.setAttribute(
+            TraceAttr.CopilotValidateOutcome,
+            CopilotValidateOutcome.InternalAuthFailed
+          )
+          span.setAttribute(TraceAttr.HttpStatusCode, 401)
+          return new NextResponse(null, { status: 401 })
+        }
 
-    const validationResult = ValidateApiKeySchema.safeParse(body)
+        const body = await req.json().catch(() => null)
+        const validationResult = ValidateApiKeySchema.safeParse(body)
+        if (!validationResult.success) {
+          logger.warn('Invalid validation request', { errors: validationResult.error.errors })
+          span.setAttribute(TraceAttr.CopilotValidateOutcome, CopilotValidateOutcome.InvalidBody)
+          span.setAttribute(TraceAttr.HttpStatusCode, 400)
+          return NextResponse.json(
+            {
+              error: 'userId is required',
+              details: validationResult.error.errors,
+            },
+            { status: 400 }
+          )
+        }
 
-    if (!validationResult.success) {
-      logger.warn('Invalid validation request', { errors: validationResult.error.errors })
-      return NextResponse.json(
-        {
-          error: 'userId is required',
-          details: validationResult.error.errors,
-        },
-        { status: 400 }
-      )
-    }
+        const { userId } = validationResult.data
+        span.setAttribute(TraceAttr.UserId, userId)
 
-    const { userId } = validationResult.data
+        const [existingUser] = await db.select().from(user).where(eq(user.id, userId)).limit(1)
+        if (!existingUser) {
+          logger.warn('[API VALIDATION] userId does not exist', { userId })
+          span.setAttribute(TraceAttr.CopilotValidateOutcome, CopilotValidateOutcome.UserNotFound)
+          span.setAttribute(TraceAttr.HttpStatusCode, 403)
+          return NextResponse.json({ error: 'User not found' }, { status: 403 })
+        }
 
-    const [existingUser] = await db.select().from(user).where(eq(user.id, userId)).limit(1)
-    if (!existingUser) {
-      logger.warn('[API VALIDATION] userId does not exist', { userId })
-      return NextResponse.json({ error: 'User not found' }, { status: 403 })
-    }
+        logger.info('[API VALIDATION] Validating usage limit', { userId })
+        const { isExceeded, currentUsage, limit } = await checkServerSideUsageLimits(userId)
+        span.setAttributes({
+          [TraceAttr.BillingUsageCurrent]: currentUsage,
+          [TraceAttr.BillingUsageLimit]: limit,
+          [TraceAttr.BillingUsageExceeded]: isExceeded,
+        })
 
-    logger.info('[API VALIDATION] Validating usage limit', { userId })
+        logger.info('[API VALIDATION] Usage limit validated', {
+          userId,
+          currentUsage,
+          limit,
+          isExceeded,
+        })
 
-    const { isExceeded, currentUsage, limit } = await checkServerSideUsageLimits(userId)
+        if (isExceeded) {
+          logger.info('[API VALIDATION] Usage exceeded', { userId, currentUsage, limit })
+          span.setAttribute(TraceAttr.CopilotValidateOutcome, CopilotValidateOutcome.UsageExceeded)
+          span.setAttribute(TraceAttr.HttpStatusCode, 402)
+          return new NextResponse(null, { status: 402 })
+        }
 
-    logger.info('[API VALIDATION] Usage limit validated', {
-      userId,
-      currentUsage,
-      limit,
-      isExceeded,
-    })
-
-    if (isExceeded) {
-      logger.info('[API VALIDATION] Usage exceeded', { userId, currentUsage, limit })
-      return new NextResponse(null, { status: 402 })
+        span.setAttribute(TraceAttr.CopilotValidateOutcome, CopilotValidateOutcome.Ok)
+        span.setAttribute(TraceAttr.HttpStatusCode, 200)
+        return new NextResponse(null, { status: 200 })
+      } catch (error) {
+        logger.error('Error validating usage limit', { error })
+        span.setAttribute(TraceAttr.CopilotValidateOutcome, CopilotValidateOutcome.InternalError)
+        span.setAttribute(TraceAttr.HttpStatusCode, 500)
+        return NextResponse.json({ error: 'Failed to validate usage' }, { status: 500 })
+      }
     }
-
-    return new NextResponse(null, { status: 200 })
-  } catch (error) {
-    logger.error('Error validating usage limit', { error })
-    return NextResponse.json({ error: 'Failed to validate usage' }, { status: 500 })
-  }
-})
+  )
+)

apps/sim/app/api/copilot/auto-allowed-tools/route.ts

@@ -2,6 +2,8 @@ import { createLogger } from '@sim/logger'
 import { type NextRequest, NextResponse } from 'next/server'
 import { getSession } from '@/lib/auth'
 import { SIM_AGENT_API_URL } from '@/lib/copilot/constants'
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { fetchGo } from '@/lib/copilot/request/go/fetch'
 import { env } from '@/lib/core/config/env'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 
@@ -31,9 +33,15 @@ export const GET = withRouteHandler(async () => {
 
     const userId = session.user.id
 
-    const res = await fetch(
+    const res = await fetchGo(
       `${SIM_AGENT_API_URL}/api/tool-preferences/auto-allowed?userId=${encodeURIComponent(userId)}`,
-      { method: 'GET', headers: copilotHeaders() }
+      {
+        method: 'GET',
+        headers: copilotHeaders(),
+        spanName: 'sim → go /api/tool-preferences/auto-allowed',
+        operation: 'list_auto_allowed_tools',
+        attributes: { [TraceAttr.UserId]: userId },
+      }
     )
 
     if (!res.ok) {
@@ -67,10 +75,13 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
       return NextResponse.json({ error: 'toolId must be a string' }, { status: 400 })
     }
 
-    const res = await fetch(`${SIM_AGENT_API_URL}/api/tool-preferences/auto-allowed`, {
+    const res = await fetchGo(`${SIM_AGENT_API_URL}/api/tool-preferences/auto-allowed`, {
       method: 'POST',
       headers: copilotHeaders(),
       body: JSON.stringify({ userId, toolId: body.toolId }),
+      spanName: 'sim → go /api/tool-preferences/auto-allowed',
+      operation: 'add_auto_allowed_tool',
+      attributes: { [TraceAttr.UserId]: userId, [TraceAttr.ToolId]: body.toolId },
     })
 
     if (!res.ok) {
@@ -108,9 +119,15 @@ export const DELETE = withRouteHandler(async (request: NextRequest) => {
       return NextResponse.json({ error: 'toolId query parameter is required' }, { status: 400 })
     }
 
-    const res = await fetch(
+    const res = await fetchGo(
       `${SIM_AGENT_API_URL}/api/tool-preferences/auto-allowed?userId=${encodeURIComponent(userId)}&toolId=${encodeURIComponent(toolId)}`,
-      { method: 'DELETE', headers: copilotHeaders() }
+      {
+        method: 'DELETE',
+        headers: copilotHeaders(),
+        spanName: 'sim → go /api/tool-preferences/auto-allowed',
+        operation: 'remove_auto_allowed_tool',
+        attributes: { [TraceAttr.UserId]: userId, [TraceAttr.ToolId]: toolId },
+      }
     )
 
     if (!res.ok) {