feat: Telemery add soft fail for client errors (#350)

* feat: update tools status as soft_fail

Author: jirispilka

src/const.ts

@@ -10,7 +10,7 @@ export const ACTOR_MAX_MEMORY_MBYTES = 4_096; // If the Actor requires 8GB of me
 // Tool output
 /**
  * Usual tool output limit is 25k tokens where 1 token =~ 4 characters
- * thus 50k chars so we have some buffer becase there was some issue with Claude code Actor call output token count.
+ * thus 50k chars so we have some buffer because there was some issue with Claude code Actor call output token count.
  * This is primarily used for Actor tool call output, but we can then
  * reuse this in other tools as well.
  */
@@ -120,10 +120,9 @@ export const TELEMETRY_ENV = {
     DEV: 'DEV',
     PROD: 'PROD',
 } as const;
-export type TelemetryEnv = (typeof TELEMETRY_ENV)[keyof typeof TELEMETRY_ENV];
 
 export const DEFAULT_TELEMETRY_ENABLED = true;
-export const DEFAULT_TELEMETRY_ENV: TelemetryEnv = TELEMETRY_ENV.PROD;
+export const DEFAULT_TELEMETRY_ENV = TELEMETRY_ENV.PROD;
 
 // We are using the same values as apify-core for consistency (despite that we ship events of different types).
 // https://github.com/apify/apify-core/blob/2284766c122c6ac5bc4f27ec28051f4057d6f9c0/src/packages/analytics/src/server/segment.ts#L28
@@ -133,6 +132,18 @@ export const SEGMENT_FLUSH_AT_EVENTS = 50;
 // Flush interval in milliseconds (default is 10000)
 export const SEGMENT_FLUSH_INTERVAL_MS = 5_000;
 
+// Tool status
+/**
+ * Unified status constants for tool execution lifecycle.
+ * Single source of truth for all tool status values.
+ */
+export const TOOL_STATUS = {
+    SUCCEEDED: 'SUCCEEDED',
+    FAILED: 'FAILED',
+    ABORTED: 'ABORTED',
+    SOFT_FAIL: 'SOFT_FAIL',
+} as const;
+
 export const SERVER_INSTRUCTIONS = `
 Apify is the world's largest marketplace of tools for web scraping, data extraction, and web automation.
 These tools are called **Actors**. They enable you to extract structured data from social media, e-commerce, search engines, maps, travel sites, and many other sources.

src/mcp/server.ts

@@ -36,7 +36,7 @@ import {
     SKYFIRE_PAY_ID_PROPERTY_DESCRIPTION,
     SKYFIRE_README_CONTENT,
     SKYFIRE_TOOL_INSTRUCTIONS,
-    type TelemetryEnv,
+    TOOL_STATUS,
 } from '../const.js';
 import { prompts } from '../prompts/index.js';
 import { getTelemetryEnv, trackToolCall } from '../telemetry.js';
@@ -47,14 +47,17 @@ import type {
     ActorsMcpServerOptions,
     ActorTool,
     HelperTool,
+    TelemetryEnv,
     ToolCallTelemetryProperties,
     ToolEntry,
+    ToolStatus,
 } from '../types.js';
 import { buildActorResponseContent } from '../utils/actor-response.js';
 import { parseBooleanFromString } from '../utils/generic.js';
 import { logHttpError } from '../utils/logging.js';
 import { buildMCPResponse } from '../utils/mcp.js';
 import { createProgressTracker } from '../utils/progress.js';
+import { getToolStatusFromError } from '../utils/tool-status.js';
 import { cloneToolEntry, getToolPublicFieldOnly } from '../utils/tools.js';
 import { getUserIdFromTokenCached } from '../utils/userid-cache.js';
 import { getPackageVersion } from '../utils/version.js';
@@ -581,7 +584,7 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool
             const { telemetryData, userId } = await this.prepareTelemetryData(tool, mcpSessionId, apifyToken);
 
             const startTime = Date.now();
-            let toolStatus: 'succeeded' | 'failed' | 'aborted' = 'succeeded';
+            let toolStatus: ToolStatus = TOOL_STATUS.SUCCEEDED;
 
             try {
                 // Handle internal tool
@@ -605,8 +608,19 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool
                     if (progressTracker) {
                         progressTracker.stop();
                     }
-                    toolStatus = ('isError' in res && res.isError) ? 'failed' : 'succeeded';
-                    return { ...res };
+
+                    // If tool provided internal status, use it; otherwise infer from isError flag
+                    const { internalToolStatus, ...rest } = res as { internalToolStatus?: ToolStatus; isError?: boolean };
+                    if (internalToolStatus !== undefined) {
+                        toolStatus = internalToolStatus;
+                    } else if ('isError' in rest && rest.isError) {
+                        toolStatus = TOOL_STATUS.FAILED;
+                    } else {
+                        toolStatus = TOOL_STATUS.SUCCEEDED;
+                    }
+
+                    // Never expose internal _toolStatus field to MCP clients
+                    return { ...rest };
                 }
 
                 if (tool.type === 'actor-mcp') {
@@ -618,8 +632,8 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool
 Please verify the server URL is correct and accessible, and ensure you have a valid Apify token with appropriate permissions.`;
                             log.softFail(msg, { statusCode: 408 }); // 408 Request Timeout
                             await this.server.sendLoggingMessage({ level: 'error', data: msg });
-                            toolStatus = 'failed';
-                            return buildMCPResponse([msg], true);
+                            toolStatus = TOOL_STATUS.SOFT_FAIL;
+                            return buildMCPResponse({ texts: [msg], isError: true });
                         }
 
                         // Only set up notification handlers if progressToken is provided by the client
@@ -649,6 +663,8 @@ Please verify the server URL is correct and accessible, and ensure you have a va
                             timeout: EXTERNAL_TOOL_CALL_TIMEOUT_MSEC,
                         });
 
+                        // For external MCP servers we do not try to infer soft_fail vs failed from isError.
+                        // We treat the call as succeeded at the telemetry layer unless an actual error is thrown.
                         return { ...res };
                     } finally {
                         if (client) await client.close();
@@ -658,7 +674,7 @@ Please verify the server URL is correct and accessible, and ensure you have a va
                 // Handle actor tool
                 if (tool.type === 'actor') {
                     if (this.options.skyfireMode && args['skyfire-pay-id'] === undefined) {
-                        return buildMCPResponse([SKYFIRE_TOOL_INSTRUCTIONS]);
+                        return buildMCPResponse({ texts: [SKYFIRE_TOOL_INSTRUCTIONS] });
                     }
 
                     // Create progress tracker if progressToken is available
@@ -686,7 +702,7 @@ Please verify the server URL is correct and accessible, and ensure you have a va
                         );
 
                         if (!callResult) {
-                            toolStatus = 'aborted';
+                            toolStatus = TOOL_STATUS.ABORTED;
                             // Receivers of cancellation notifications SHOULD NOT send a response for the cancelled request
                             // https://modelcontextprotocol.io/specification/2025-06-18/basic/utilities/cancellation#behavior-requirements
                             return { };
@@ -700,14 +716,17 @@ Please verify the server URL is correct and accessible, and ensure you have a va
                         }
                     }
                 }
+                // If we reached here without returning, it means the tool type was not recognized (user error)
+                toolStatus = TOOL_STATUS.SOFT_FAIL;
             } catch (error) {
-                toolStatus = extra.signal?.aborted ? 'aborted' : 'failed';
+                toolStatus = getToolStatusFromError(error, Boolean(extra.signal?.aborted));
                 logHttpError(error, 'Error occurred while calling tool', { toolName: name });
                 const errorMessage = (error instanceof Error) ? error.message : 'Unknown error';
-                return buildMCPResponse([
-                    `Error calling tool "${name}": ${errorMessage}.
-Please verify the tool name, input parameters, and ensure all required resources are available.`,
-                ], true);
+                return buildMCPResponse({
+                    texts: [`Error calling tool "${name}": ${errorMessage}.  Please verify the tool name, input parameters, and ensure all required resources are available.`],
+                    isError: true,
+                    toolStatus,
+                });
             } finally {
                 this.finalizeAndTrackTelemetry(telemetryData, userId, startTime, toolStatus);
             }
@@ -735,13 +754,13 @@ Please verify the tool name and ensure the tool is properly registered.`;
      * @param telemetryData - Telemetry data to finalize and track (null if telemetry is disabled)
      * @param userId - Apify user ID (string or null if not available)
      * @param startTime - Timestamp when the tool call started
-     * @param toolStatus - Final status of the tool call ('succeeded', 'failed', or 'aborted')
+     * @param toolStatus - Final status of the tool call
      */
     private finalizeAndTrackTelemetry(
         telemetryData: ToolCallTelemetryProperties | null,
         userId: string | null,
         startTime: number,
-        toolStatus: 'succeeded' | 'failed' | 'aborted',
+        toolStatus: ToolStatus,
     ): void {
         if (!telemetryData) {
             return;
@@ -787,7 +806,7 @@ Please verify the tool name and ensure the tool is properly registered.`;
             mcp_session_id: mcpSessionId || '',
             transport_type: this.options.transportType || '',
             tool_name: toolFullName,
-            tool_status: 'succeeded', // Will be updated in finally
+            tool_status: TOOL_STATUS.SUCCEEDED, // Will be updated in finally
             tool_exec_time_ms: 0, // Will be calculated in finally
         };
 

src/stdio.ts

@@ -29,11 +29,11 @@ import { hideBin } from 'yargs/helpers';
 import log from '@apify/log';
 
 import { ApifyClient } from './apify-client.js';
-import { DEFAULT_TELEMETRY_ENV, TELEMETRY_ENV, type TelemetryEnv } from './const.js';
+import { DEFAULT_TELEMETRY_ENV, TELEMETRY_ENV } from './const.js';
 import { processInput } from './input.js';
 import { ActorsMcpServer } from './mcp/server.js';
 import { getTelemetryEnv } from './telemetry.js';
-import type { Input, ToolSelector } from './types.js';
+import type { Input, TelemetryEnv, ToolSelector } from './types.js';
 import { parseCommaSeparatedList } from './utils/generic.js';
 import { loadToolsFromInput } from './utils/tools-loader.js';
 

src/telemetry.ts

@@ -9,9 +9,8 @@ import {
     SEGMENT_FLUSH_AT_EVENTS,
     SEGMENT_FLUSH_INTERVAL_MS,
     TELEMETRY_ENV,
-    type TelemetryEnv,
 } from './const.js';
-import type { ToolCallTelemetryProperties } from './types.js';
+import type { TelemetryEnv, ToolCallTelemetryProperties } from './types.js';
 
 const DEV_WRITE_KEY = '9rPHlMtxX8FJhilGEwkfUoZ0uzWxnzcT';
 const PROD_WRITE_KEY = 'cOkp5EIJaN69gYaN8bcp7KtaD0fGABwJ';

src/tools/actor.ts

@@ -6,15 +6,14 @@ import zodToJsonSchema from 'zod-to-json-schema';
 import log from '@apify/log';
 
 import { ApifyClient } from '../apify-client.js';
-import {
-    ACTOR_MAX_MEMORY_MBYTES,
+import { ACTOR_MAX_MEMORY_MBYTES,
     CALL_ACTOR_MCP_MISSING_TOOL_NAME_MSG,
     HelperTools,
     RAG_WEB_BROWSER,
     RAG_WEB_BROWSER_ADDITIONAL_DESC,
     SKYFIRE_TOOL_INSTRUCTIONS,
     TOOL_MAX_OUTPUT_CHARS,
-} from '../const.js';
+    TOOL_STATUS } from '../const.js';
 import { getActorMCPServerPath, getActorMCPServerURL } from '../mcp/actors.js';
 import { connectMCPClient } from '../mcp/client.js';
 import { getMCPServerTools } from '../mcp/proxy.js';
@@ -408,9 +407,11 @@ EXAMPLES:
 
         // Standby Actors, thus MCPs, are not supported in Skyfire mode
         if (isActorMcpServer && apifyMcpServer.options.skyfireMode) {
-            return buildMCPResponse([
-                `This Actor (${actorName}) is an MCP server and cannot be accessed using a Skyfire token. To use this Actor, please provide a valid Apify token instead of a Skyfire token.`,
-            ], true);
+            return buildMCPResponse({
+                texts: [`This Actor (${actorName}) is an MCP server and cannot be accessed using a Skyfire token. To use this Actor, please provide a valid Apify token instead of a Skyfire token.`],
+                isError: true,
+                toolStatus: TOOL_STATUS.SOFT_FAIL,
+            });
         }
 
         try {
@@ -423,24 +424,34 @@ EXAMPLES:
                     try {
                         client = await connectMCPClient(mcpServerUrl, apifyToken);
                         if (!client) {
-                            return buildMCPResponse([`Failed to connect to MCP server ${mcpServerUrl}`], true);
+                            return buildMCPResponse({
+                                texts: [`Failed to connect to MCP server ${mcpServerUrl}`],
+                                isError: true,
+                                toolStatus: TOOL_STATUS.SOFT_FAIL,
+                            });
                         }
                         const toolsResponse = await client.listTools();
 
                         const toolsInfo = toolsResponse.tools.map((tool) => `**${tool.name}**\n${tool.description || 'No description'}\nInput schema:\n\`\`\`json\n${JSON.stringify(tool.inputSchema)}\n\`\`\``,
                         ).join('\n\n');
 
-                        return buildMCPResponse([`This is an MCP Server Actor with the following tools:\n\n${toolsInfo}\n\nTo call a tool, use step="call" with actor name format: "${baseActorName}:{toolName}"`]);
+                        return buildMCPResponse({
+                            texts: [`This is an MCP Server Actor with the following tools:\n\n${toolsInfo}\n\nTo call a tool, use step="call" with actor name format: "${baseActorName}:{toolName}"`],
+                        });
                     } finally {
                         if (client) await client.close();
                     }
                 } else {
-                    // Regular actor: return schema
+                    // Regular Actor: return schema
                     const details = await fetchActorDetails(apifyClientForDefinition, baseActorName);
                     if (!details) {
-                        return buildMCPResponse([`Actor information for '${baseActorName}' was not found.
+                        return buildMCPResponse({
+                            texts: [`Actor information for '${baseActorName}' was not found.
 Please verify Actor ID or name format (e.g., "username/name" like "apify/rag-web-browser") and ensure that the Actor exists.
-You can search for available Actors using the tool: ${HelperTools.STORE_SEARCH}.`], true);
+You can search for available Actors using the tool: ${HelperTools.STORE_SEARCH}.`],
+                            isError: true,
+                            toolStatus: TOOL_STATUS.SOFT_FAIL,
+                        });
                     }
                     const content = [
                         `Actor name: ${actorName}`,
@@ -452,7 +463,7 @@ You can search for available Actors using the tool: ${HelperTools.STORE_SEARCH}.
                     if (apifyMcpServer.options.skyfireMode) {
                         content.push(SKYFIRE_TOOL_INSTRUCTIONS);
                     }
-                    return buildMCPResponse(content);
+                    return buildMCPResponse({ texts: content });
                 }
             }
 
@@ -478,28 +489,41 @@ You can search for available Actors using the tool: ${HelperTools.STORE_SEARCH}.
 
             // Step 2: Call the Actor
             if (!input) {
-                return buildMCPResponse([`Input is required when step="call". Please provide the input parameter based on the Actor's input schema.`], true);
+                // Missing input is most likely an LLM error, so NOT marking as a soft-fail to track potential issues with tool description
+                return buildMCPResponse({
+                    texts: [`Input is required when step="call". Please provide the input parameter based on the Actor's input schema.`],
+                    isError: true,
+                });
             }
 
             // Handle the case where LLM does not respect instructions when calling MCP server Actors
             // and does not provide the tool name.
             const isMcpToolNameInvalid = mcpToolName === undefined || mcpToolName.trim().length === 0;
             if (isActorMcpServer && isMcpToolNameInvalid) {
-                return buildMCPResponse([CALL_ACTOR_MCP_MISSING_TOOL_NAME_MSG], true);
+                return buildMCPResponse({
+                    texts: [CALL_ACTOR_MCP_MISSING_TOOL_NAME_MSG],
+                    isError: true,
+                });
             }
 
             // Handle MCP tool calls
             if (mcpToolName) {
                 if (!isActorMcpServer) {
-                    return buildMCPResponse([`Actor '${baseActorName}' is not an MCP server.`], true);
+                    return buildMCPResponse({
+                        texts: [`Actor '${baseActorName}' is not an MCP server.`],
+                        isError: true,
+                    });
                 }
 
                 const mcpServerUrl = mcpServerUrlOrFalse;
                 let client: Client | null = null;
                 try {
                     client = await connectMCPClient(mcpServerUrl, apifyToken);
                     if (!client) {
-                        return buildMCPResponse([`Failed to connect to MCP server ${mcpServerUrl}`], true);
+                        return buildMCPResponse({
+                            texts: [`Failed to connect to MCP server ${mcpServerUrl}`],
+                            isError: true,
+                        });
                     }
 
                     const result = await client.callTool({
@@ -517,9 +541,13 @@ You can search for available Actors using the tool: ${HelperTools.STORE_SEARCH}.
             const [actor] = await getActorsAsTools([actorName], apifyClient);
 
             if (!actor) {
-                return buildMCPResponse([`Actor '${actorName}' was not found.
+                return buildMCPResponse({
+                    texts: [`Actor '${actorName}' was not found.
 Please verify Actor ID or name format (e.g., "username/name" like "apify/rag-web-browser") and ensure that the Actor exists.
-You can search for available Actors using the tool: ${HelperTools.STORE_SEARCH}.`], true);
+You can search for available Actors using the tool: ${HelperTools.STORE_SEARCH}.`],
+                    isError: true,
+                    toolStatus: TOOL_STATUS.SOFT_FAIL,
+                });
             }
 
             if (!actor.ajvValidate(input)) {
@@ -531,7 +559,7 @@ You can search for available Actors using the tool: ${HelperTools.STORE_SEARCH}.
                 if (errors && errors.length > 0) {
                     content.push(`Validation errors: ${errors.map((e) => (e as { message?: string }).message).join(', ')}`);
                 }
-                return buildMCPResponse(content);
+                return buildMCPResponse({ texts: content, isError: true });
             }
 
             const callResult = await callActorGetDataset(
@@ -554,9 +582,13 @@ You can search for available Actors using the tool: ${HelperTools.STORE_SEARCH}.
             return { content };
         } catch (error) {
             logHttpError(error, 'Failed to call Actor', { actorName, performStep });
-            return buildMCPResponse([`Failed to call Actor '${actorName}': ${error instanceof Error ? error.message : String(error)}.
+            // Let the server classify the error; we only mark it as an MCP error response
+            return buildMCPResponse({
+                texts: [`Failed to call Actor '${actorName}': ${error instanceof Error ? error.message : String(error)}.
 Please verify the Actor name, input parameters, and ensure the Actor exists.
-You can search for available Actors using the tool: ${HelperTools.STORE_SEARCH}, or get Actor details using: ${HelperTools.ACTOR_GET_DETAILS}.`], true);
+You can search for available Actors using the tool: ${HelperTools.STORE_SEARCH}, or get Actor details using: ${HelperTools.ACTOR_GET_DETAILS}.`],
+                isError: true,
+            });
         }
     },
 };