Raw schema
{
"type": "object",
"properties": {
"videoUrl": {
"type": "string",
"description": "Direct http(s) URL to a raw media file. Share links (YouTube/Drive/Dropbox) are not accepted."
},
"assetId": {
"type": "string",
"minLength": 1,
"description": "ID of a previously uploaded asset. Must be in status 'ready' before this call."
},
"base64Video": {
"type": "string",
"minLength": 1,
"description": "Base64-encoded video data, max 30 MB. Prefer videoUrl or assetId for larger inputs."
},
"modelName": {
"type": "string",
"enum": [
"pegasus1.2",
"pegasus1.5"
],
"description": "Defaults to pegasus1.5 (handles both prompt-only general analysis and time_based_metadata segmentation). pegasus1.2 is retained for backwards compatibility — set it explicitly only if the caller has an existing pegasus1.2 workflow they need to preserve. Pegasus 1.2 is on a deprecation path; new integrations should use 1.5."
},
"prompt": {
"type": "string",
"description": "Natural-language prompt for general-mode analysis (max 2000 tokens). Not allowed with analysisMode=time_based_metadata. Mutually exclusive with promptV2."
},
"promptV2": {
"type": "object",
"properties": {
"inputText": {
"type": "string",
"minLength": 1,
"maxLength": 2000
},
"mediaSources": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string",
"minLength": 1
},
"mediaType": {
"type": "string",
"const": "image"
},
"url": {
"type": "string"
},
"assetId": {
"type": "string"
},
"base64String": {
"type": "string"
}
},
"required": [
"name",
"mediaType"
],
"additionalProperties": false
},
"minItems": 1,
"maxItems": 4
}
},
"required": [
"inputText"
],
"additionalProperties": false,
"description": "Structured prompt with <@name> image references (1.2.4+, Pegasus 1.5 only). Mutually exclusive with prompt. Not allowed with analysisMode=time_based_metadata."
},
"temperature": {
"type": "number",
"minimum": 0,
"maximum": 1
},
"maxTokens": {
"type": "integer",
"exclusiveMinimum": 0,
"description": "Max tokens to generate. pegasus1.2: 1-4096. pegasus1.5: 2048-32768 (default 32768)."
},
"analysisMode": {
"type": "string",
"const": "time_based_metadata",
"description": "Set to 'time_based_metadata' for Pegasus 1.5 segmentation. Requires segmentDefinitions."
},
"segmentDefinitions": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {
"type": "string",
"minLength": 1
},
"description": {
"type": "string",
"minLength": 1
},
"fields": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string",
"minLength": 1
},
"type": {
"type": "string",
"enum": [
"string",
"boolean",
"number",
"integer",
"array"
]
},
"description": {
"type": "string",
"minLength": 1
},
"enum": {
"type": "array",
"items": {
"type": "string"
},
"maxItems": 50
},
"items": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": [
"string",
"number",
"boolean",
"integer"
]
}
},
"required": [
"type"
],
"additionalProperties": false
}
},
"required": [
"name",
"type",
"description"
],
"additionalProperties": false
},
"maxItems": 20
},
"mediaSources": {
"type": "array",
"items": {
"$ref": "#/properties/promptV2/properties/mediaSources/items"
},
"maxItems": 4
},
"timeRanges": {
"type": "array",
"items": {
"type": "object",
"properties": {
"startTime": {
"type": "number",
"minimum": 0
},
"endTime": {
"type": "number",
"minimum": 0
}
},
"required": [
"startTime",
"endTime"
],
"additionalProperties": false
},
"description": "Time windows that restrict segment extraction. Pegasus 1.5 + analysisMode=time_based_metadata only. Each range needs end_time - start_time ≥ 2s. Mutually exclusive with top-level startTime/endTime and with min/maxSegmentDuration."
}
},
"required": [
"id",
"description"
],
"additionalProperties": false
},
"minItems": 1,
"maxItems": 10,
"description": "Segment definitions for time_based_metadata mode. Each includes custom fields, optional reference images, and optional timeRanges."
},
"jsonSchema": {
"type": "object",
"additionalProperties": {},
"description": "JSON Schema (Draft 2020-12 subset) for structured JSON output. Cannot be combined with segmentDefinitions."
},
"minSegmentDuration": {
"type": "number",
"minimum": 2,
"description": "Minimum seconds per segment (time_based_metadata only). Mutually exclusive with segmentDefinition.timeRanges."
},
"maxSegmentDuration": {
"type": "number",
"minimum": 2,
"description": "Maximum seconds per segment (time_based_metadata only). Mutually exclusive with segmentDefinition.timeRanges."
},
"startTime": {
"type": "number",
"minimum": 0,
"description": "Start of the analysis window in seconds (1.2.4+, pegasus1.5). Mutually exclusive with segmentDefinition.timeRanges. end - start must be ≥4s."
},
"endTime": {
"type": "number",
"minimum": 0,
"description": "End of the analysis window in seconds (1.2.4+, pegasus1.5)."
},
"customId": {
"type": "string",
"minLength": 1,
"maxLength": 64,
"pattern": "^[A-Za-z0-9_-]+$",
"description": "Caller-supplied identifier (1.2.4+). 1-64 chars, alphanumeric / hyphen / underscore. Surfaced back in get-analyse-task and list-analyse-tasks responses."
},
"user_intent": {
"type": "string",
"description": "A concise summary of what the user is trying to accomplish, derived from their message or the\nconversation context that triggered this tool call.\nThis is used to understand the user's intent and context to improve the overall user experience.\n\n- For short, self-contained prompts (e.g. \"I want new shoes\"), copy the user message as-is.\n- For longer conversations or detailed requests, summarize the core goal and any relevant\n context in 1-2 sentences. Focus on intent, constraints, and preferences - not the full\n dialogue.\n\nBefore sending, strip all personally identifiable information (PII), including but not\nlimited to:\n - Names (first, last, usernames, handles)\n - Email addresses\n - Phone numbers\n - Physical addresses (street, city, zip/postal code, country when tied to an individual)\n - Dates of birth or exact ages\n - Government-issued ID numbers (SSN, passport, driver's license, etc.)\n - Payment or financial information (card numbers, bank accounts, etc.)\n - IP addresses or device identifiers\n - Account credentials (passwords, tokens, API keys)\n - Health or biometric data\n - Any other information that could identify a specific individual\n\nReplace stripped values with a generic placeholder (e.g. \"[name]\", \"[email]\", \"[address]\").\n\nExamples:\n User: \"I want red running shoes under $100\"\n -> \"I want red running shoes under $100\"\n\n User: \"Hi, I'm John Smith, john@example.com, and I'm looking for flights from Paris to\n Tokyo for 2 adults departing around mid-June, budget around EUR2000 total\"\n -> \"Looking for flights from Paris to Tokyo for 2 adults, mid-June, budget ~EUR2000\"\n\n User: \"I need help resetting my password for account ID acct_12345\"\n -> \"I need help resetting my password for account ID [account_id]\""
}
},
"additionalProperties": false,
"$schema": "http://json-schema.org/draft-07/schema#"
}