feat(gpt-apps): add tools and widget descriptors

Author: jakcinmarina

README.md

@@ -243,6 +243,39 @@ As above, this exposes only the specified Actor (`apify/my-actor`) as a tool. No
 >
 > **For production use and stable interfaces, always explicitly specify the `tools` parameter** to ensure your configuration remains consistent across updates.
 
+### UI mode configuration
+
+The `uiMode` parameter enables OpenAI-specific widget rendering in tool responses. When enabled, tools like `search-actors` return interactive widget responses optimized for OpenAI clients.
+
+**Configuring the hosted server:**
+
+Enable UI mode using the `ui` query parameter:
+
+```
+https://mcp.apify.com?ui=openai
+```
+
+You can combine it with other parameters:
+
+```
+https://mcp.apify.com?tools=actors,docs&ui=openai
+```
+
+**Configuring the CLI:**
+
+The CLI can be configured using command-line flags. For example, to enable UI mode:
+
+```bash
+npx @apify/actors-mcp-server --uiMode openai
+```
+
+You can also set it via the `UI_MODE` environment variable:
+
+```bash
+export UI_MODE=openai
+npx @apify/actors-mcp-server
+```
+
 ### Backward compatibility
 
 The v2 configuration preserves backward compatibility with v1 usage. Notes:

eslint.config.mjs

@@ -27,6 +27,7 @@ export default [
             'evals/*.ts', // Top-level evaluation scripts
             'evals/*.md', // Documentation files
             'evals/*.json', // Test case data files
+            'src/web/**', // Web directory has its own TypeScript project and build system
         ],
     },
     // Apply the shared Apify TypeScript ESLint configuration

src/actor/server.ts

@@ -15,7 +15,7 @@ import log from '@apify/log';
 
 import { ApifyClient } from '../apify-client.js';
 import { ActorsMcpServer } from '../mcp/server.js';
-import type { ApifyRequestParams } from '../types.js';
+import type { ApifyRequestParams, UiMode } from '../types.js';
 import { parseBooleanFromString } from '../utils/generic.js';
 import { getHelpMessage, HEADER_READINESS_PROBE, Routes, TransportType } from './const.js';
 import { getActorRunData } from './utils.js';
@@ -90,13 +90,17 @@ export function createExpressApp(
                 ?? parseBooleanFromString(process.env.TELEMETRY_ENABLED)
                 ?? true;
 
+            const uiModeParam = urlParams.get('ui') as UiMode | undefined;
+            const uiMode = uiModeParam ?? process.env.UI_MODE as UiMode | undefined;
+
             const mcpServer = new ActorsMcpServer({
                 taskStore,
                 setupSigintHandler: false,
                 transportType: 'sse',
                 telemetry: {
                     enabled: telemetryEnabled,
                 },
+                uiMode,
             });
             const transport = new SSEServerTransport(Routes.MESSAGE, res);
 
@@ -214,6 +218,9 @@ export function createExpressApp(
                     ?? parseBooleanFromString(process.env.TELEMETRY_ENABLED)
                     ?? true;
 
+                const uiModeParam = urlParams.get('ui') as UiMode | undefined;
+                const uiMode = uiModeParam ?? process.env.UI_MODE as UiMode | undefined;
+
                 const mcpServer = new ActorsMcpServer({
                     taskStore,
                     setupSigintHandler: false,
@@ -222,6 +229,7 @@ export function createExpressApp(
                     telemetry: {
                         enabled: telemetryEnabled,
                     },
+                    uiMode,
                 });
 
                 // Load MCP server tools

src/const.ts

@@ -44,6 +44,9 @@ export enum HelperTools {
     DOCS_SEARCH = 'search-apify-docs',
     DOCS_FETCH = 'fetch-apify-docs',
     GET_HTML_SKELETON = 'get-html-skeleton',
+    CALL_ACTOR_WIDGET = 'call-actor-widget',
+    GET_ACTOR_RUN_STATUS = 'get-actor-run-status',
+    FETCH_ACTOR_DETAILS_WIDGET = 'fetch-actor-details-widget',
 }
 
 export const RAG_WEB_BROWSER = 'apify/rag-web-browser';
@@ -227,4 +230,6 @@ These tools are called **Actors**. They enable you to extract structured data fr
   \`${HelperTools.STORE_SEARCH}\` finds robust and reliable Actors for specific websites; ${RAG_WEB_BROWSER} is a general and versatile web scraping tool.
 - **Dedicated Actor tools (e.g. ${RAG_WEB_BROWSER}) vs ${HelperTools.ACTOR_CALL}:**
   Prefer dedicated tools when available; use \`${HelperTools.ACTOR_CALL}\` only when no specialized tool exists in Apify store.
+- **Async vs sync Actor tools (${HelperTools.ACTOR_CALL} vs ${HelperTools.CALL_ACTOR_WIDGET}):**
+  Default to \`${HelperTools.ACTOR_CALL}\` (synchronous, no widget) when the user asks to “run/call” and does not request background/progress/UI. Use \`${HelperTools.CALL_ACTOR_WIDGET}\` only when the user wants background/progress/UI. After starting an async run and obtaining runId, do NOT start another async run—only poll with \`${HelperTools.GET_ACTOR_RUN_STATUS}\` using that runId.
 `;

src/mcp/server.ts

@@ -431,43 +431,160 @@ export class ActorsMcpServer {
 
     private setupResourceHandlers(): void {
         this.server.setRequestHandler(ListResourcesRequestSchema, async () => {
+            const resources = [];
+
             /**
-             * Return the usage guide resource only if Skyfire mode is enabled. No resources otherwise for normal mode.
+             * Return the usage guide resource only if Skyfire mode is enabled.
              */
             if (this.options.skyfireMode) {
-                return {
-                    resources: [
-                        {
-                            uri: 'file://readme.md',
-                            name: 'readme',
-                            description: `Apify MCP Server usage guide. Read this to understand how to use the server, especially in Skyfire mode before interacting with it.`,
-                            mimeType: 'text/markdown',
+                resources.push({
+                    uri: 'file://readme.md',
+                    name: 'readme',
+                    description: `Apify MCP Server usage guide. Read this to understand how to use the server, especially in Skyfire mode before interacting with it.`,
+                    mimeType: 'text/markdown',
+                });
+            }
+
+            if (this.options.uiMode === 'openai') {
+                resources.push({
+                    uri: 'ui://widget/search-actors.html',
+                    name: 'search-actors-widget',
+                    description: 'Interactive Actor search results widget',
+                    mimeType: 'text/html+skybridge',
+                    _meta: {
+                        'openai/outputTemplate': 'ui://widget/search-actors.html',
+                        'openai/toolInvocation/invoking': 'Searching Apify Store...',
+                        'openai/toolInvocation/invoked': 'Found Actors matching your criteria',
+                        'openai/widgetAccessible': true,
+                        'openai/resultCanProduceWidget': true,
+                        // TODO: replace with real CSP domains
+                        'openai/widgetCSP': {
+                            connect_domains: ['https://api.example.com'],
+                            resource_domains: ['https://persistent.oaistatic.com'],
                         },
-                    ],
-                };
+                        'openai/widgetDomain': 'https://chatgpt.com',
+                    },
+                });
+
+                resources.push({
+                    uri: 'ui://widget/actor-run.html',
+                    name: 'actor-run-widget',
+                    description: 'Interactive Actor run widget',
+                    mimeType: 'text/html+skybridge',
+                    _meta: {
+                        'openai/outputTemplate': 'ui://widget/actor-run.html',
+                        'openai/widgetAccessible': true,
+                        'openai/resultCanProduceWidget': true,
+                        // TODO: replace with real CSP domains
+                        'openai/widgetCSP': {
+                            connect_domains: ['https://api.example.com'],
+                            resource_domains: ['https://persistent.oaistatic.com'],
+                        },
+                        'openai/widgetDomain': 'https://chatgpt.com',
+                    },
+                });
             }
-            return { resources: [] };
+
+            return { resources };
         });
 
-        if (this.options.skyfireMode) {
-            this.server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
-                const { uri } = request.params;
-                if (uri === 'file://readme.md') {
+        this.server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
+            const { uri } = request.params;
+            if (this.options.skyfireMode && uri === 'file://readme.md') {
+                return {
+                    contents: [{
+                        uri: 'file://readme.md',
+                        mimeType: 'text/markdown',
+                        text: SKYFIRE_README_CONTENT,
+                    }],
+                };
+            }
+
+            if (this.options.uiMode === 'openai' && uri.startsWith('ui://widget/')) {
+                try {
+                    log.debug('Reading widget files', { uri });
+                    const fs = await import('node:fs');
+                    const path = await import('node:path');
+                    const { fileURLToPath } = await import('node:url');
+
+                    // Get the directory of this file
+                    const filename = fileURLToPath(import.meta.url);
+                    const dirName = path.dirname(filename);
+
+                    let widgetJsFilename = '';
+                    let widgetTitle = '';
+
+                    if (uri === 'ui://widget/search-actors.html') {
+                        widgetJsFilename = 'search-actors-widget.js';
+                        widgetTitle = 'Apify Actor Search';
+                    } else if (uri === 'ui://widget/actor-run.html') {
+                        widgetJsFilename = 'actor-run-widget.js';
+                        widgetTitle = 'Apify Actor Run';
+                    } else {
+                        return {
+                            contents: [{
+                                uri, mimeType: 'text/plain', text: `Widget resource ${uri} not found`,
+                            }],
+                        };
+                    }
+
+                    const widgetJsPath = path.resolve(dirName, `../web/dist/${widgetJsFilename}`);
+
+                    log.debug('Reading widget file', { widgetJsPath });
+
+                    const widgetJs = fs.readFileSync(widgetJsPath, 'utf-8');
+
+                    const widgetHtml = `<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>${widgetTitle}</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module">${widgetJs}</script>
+  </body>
+</html>`;
+
                     return {
                         contents: [{
-                            uri: 'file://readme.md',
-                            mimeType: 'text/markdown',
-                            text: SKYFIRE_README_CONTENT,
+                            uri,
+                            mimeType: 'text/html+skybridge',
+                            text: widgetHtml,
+                            html: widgetHtml,
+                            _meta: {
+                                'openai/widgetPrefersBorder': true,
+                                'openai/outputTemplate': uri,
+                                'openai/widgetAccessible': true,
+                                'openai/resultCanProduceWidget': true,
+                                // TODO: replace with real CSP domains
+                                'openai/widgetCSP': {
+                                    connect_domains: ['https://api.example.com'],
+                                    resource_domains: ['https://persistent.oaistatic.com'],
+                                },
+                                'openai/widgetDomain': 'https://chatgpt.com',
+                            },
+                        }],
+                    };
+                } catch (error) {
+                    const errorMessage = error instanceof Error ? error.message : String(error);
+                    return {
+                        contents: [{
+                            uri,
+                            mimeType: 'text/plain',
+                            text: `Failed to load widget: ${errorMessage}`,
                         }],
                     };
                 }
-                return {
-                    contents: [{
-                        uri, mimeType: 'text/plain', text: `Resource ${uri} not found`,
-                    }],
-                };
-            });
-        }
+            }
+
+            return {
+                contents: [{
+                    uri, mimeType: 'text/plain', text: `Resource ${uri} not found`,
+                }],
+            };
+        });
 
         this.server.setRequestHandler(ListResourceTemplatesRequestSchema, async () => {
             // No resource templates available, return empty response
@@ -616,7 +733,10 @@ export class ActorsMcpServer {
          * @returns {object} - The response object containing the tools.
          */
         this.server.setRequestHandler(ListToolsRequestSchema, async () => {
-            const tools = Array.from(this.tools.values()).map((tool) => getToolPublicFieldOnly(tool));
+            const tools = Array.from(this.tools.values()).map((tool) => getToolPublicFieldOnly(tool, {
+                uiMode: this.options.uiMode,
+                filterOpenAiMeta: true,
+            }));
             return { tools };
         });
 

src/stdio.ts

@@ -33,7 +33,7 @@ import { DEFAULT_TELEMETRY_ENV, TELEMETRY_ENV } from './const.js';
 import { processInput } from './input.js';
 import { ActorsMcpServer } from './mcp/server.js';
 import { getTelemetryEnv } from './telemetry.js';
-import type { ApifyRequestParams, Input, TelemetryEnv, ToolSelector } from './types.js';
+import type { ApifyRequestParams, Input, TelemetryEnv, ToolSelector, UiMode } from './types.js';
 import { parseCommaSeparatedList } from './utils/generic.js';
 import { loadToolsFromInput } from './utils/tools-loader.js';
 
@@ -53,6 +53,10 @@ type CliArgs = {
     telemetryEnabled: boolean;
     /** Telemetry environment: 'PROD' or 'DEV' (default: 'PROD', only used when telemetry-enabled is true) */
     telemetryEnv: TelemetryEnv;
+    /** UI mode for tool responses.
+     * - 'openai': OpenAI specific widget rendering. If not specified, there will be no widget rendering.
+     */
+    uiMode: UiMode;
 }
 
 /**
@@ -118,6 +122,14 @@ Default: true (enabled)`,
 - 'PROD': Send events to production Segment workspace (default)
 - 'DEV': Send events to development Segment workspace
 Only used when --telemetry-enabled is true`,
+    })
+    .option('uiMode', {
+        type: 'string',
+        choices: ['openai'],
+        default: undefined,
+        describe: `UI mode for tool responses. Can also be set via UI_MODE environment variable.
+- 'openai': OpenAI specific widget rendering
+Default: undefined (no widget rendering)`,
     })
     .help('help')
     .alias('h', 'help')
@@ -161,6 +173,7 @@ async function main() {
             env: getTelemetryEnv(argv.telemetryEnv),
         },
         token: apifyToken,
+        uiMode: argv.uiMode,
     });
 
     // Create an Input object from CLI arguments