Expose content array alongside structuredContent in workflow templates (#4384)

Make embedded resource content accessible from composite tool templates

ContentArrayToMap previously ignored EmbeddedResource content types,
making it impossible for composite tool workflows to chain on embedded
resource data (e.g., an SBOM returned by get_referrer_content).

Handle ContentTypeResource in ContentArrayToMap with "resource",
"resource_1", "resource_2" key pattern. Expose the content array as
a separate template namespace ({{.steps.X.content.*}}) alongside the
existing output namespace ({{.steps.X.output.*}}), keeping
structuredContent clean for outputSchema validation.

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: JAORMX

pkg/vmcp/composer/composer.go

@@ -13,6 +13,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/stacklok/toolhive/pkg/vmcp"
 	"github.com/stacklok/toolhive/pkg/vmcp/config"
 )
 
@@ -212,9 +213,14 @@ type StepResult struct {
 	// Status is the step status.
 	Status StepStatusType
 
-	// Output contains the step output data.
+	// Output contains the step output data (from StructuredContent or ContentArrayToMap fallback).
 	Output map[string]any
 
+	// Content holds the raw content array from the tool call result.
+	// This is exposed separately in templates via {{.steps.stepID.content.*}} so that
+	// structuredContent remains clean for outputSchema validation.
+	Content []vmcp.Content
+
 	// Error contains error information if the step failed.
 	Error error
 

pkg/vmcp/composer/template_expander.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/stacklok/toolhive/pkg/templates"
+	"github.com/stacklok/toolhive/pkg/vmcp/conversion"
 )
 
 const (
@@ -163,7 +164,10 @@ func (e *defaultTemplateExpander) expandString(
 }
 
 // buildStepsContext converts StepResult map to a template-friendly structure.
-// This provides access to step outputs via {{.steps.stepid.output.field}}.
+// This provides access to step outputs via:
+//   - {{.steps.stepid.output.field}} for structuredContent fields
+//   - {{.steps.stepid.content.text}} for text content from the content array
+//   - {{.steps.stepid.content.resource}} for embedded resource content from the content array
 func (*defaultTemplateExpander) buildStepsContext(workflowCtx *WorkflowContext) map[string]any {
 	// Acquire read lock to safely access Steps map during concurrent execution
 	workflowCtx.mu.RLock()
@@ -173,8 +177,9 @@ func (*defaultTemplateExpander) buildStepsContext(workflowCtx *WorkflowContext)
 
 	for stepID, result := range workflowCtx.Steps {
 		stepData := map[string]any{
-			"status": string(result.Status),
-			"output": result.Output,
+			"status":  string(result.Status),
+			"output":  result.Output,
+			"content": conversion.ContentArrayToMap(result.Content),
 		}
 
 		// Add error information if step failed

pkg/vmcp/composer/template_expander_test.go

@@ -10,6 +10,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/stacklok/toolhive/pkg/vmcp"
 )
 
 func TestTemplateExpander_Expand(t *testing.T) {
@@ -74,6 +76,40 @@ func TestTemplateExpander_Expand(t *testing.T) {
 			params:   map[string]any{},
 			expected: map[string]any{"val": "<no value>"},
 		},
+		{
+			name: "step content resource substitution",
+			data: map[string]any{"sbom": "{{.steps.fetch.content.resource}}"},
+			steps: map[string]*StepResult{
+				"fetch": {
+					Status: StepStatusCompleted,
+					Output: map[string]any{"format": "spdx"},
+					Content: []vmcp.Content{
+						{Type: vmcp.ContentTypeResource, Text: `{"spdxVersion":"SPDX-2.3"}`, URI: "file://sbom.json"},
+					},
+				},
+			},
+			expected: map[string]any{"sbom": `{"spdxVersion":"SPDX-2.3"}`},
+		},
+		{
+			name: "step output and content are independent namespaces",
+			data: map[string]any{
+				"format": "{{.steps.fetch.output.format}}",
+				"text":   "{{.steps.fetch.content.text}}",
+			},
+			steps: map[string]*StepResult{
+				"fetch": {
+					Status: StepStatusCompleted,
+					Output: map[string]any{"format": "spdx", "text": "structured text"},
+					Content: []vmcp.Content{
+						{Type: vmcp.ContentTypeText, Text: "content array text"},
+					},
+				},
+			},
+			expected: map[string]any{
+				"format": "spdx",
+				"text":   "content array text",
+			},
+		},
 	}
 
 	expander := NewTemplateExpander()
@@ -154,7 +190,7 @@ func TestWorkflowContext_Lifecycle(t *testing.T) {
 	assert.Equal(t, StepStatusRunning, ctx.Steps["s1"].Status)
 
 	time.Sleep(10 * time.Millisecond)
-	ctx.RecordStepSuccess("s1", map[string]any{"result": "ok"})
+	ctx.RecordStepSuccess("s1", map[string]any{"result": "ok"}, nil)
 	assert.Equal(t, StepStatusCompleted, ctx.Steps["s1"].Status)
 	assert.Greater(t, ctx.Steps["s1"].Duration, time.Duration(0))
 
@@ -185,12 +221,12 @@ func TestWorkflowContext_GetLastStepOutput(t *testing.T) {
 	// Add steps with different completion times
 	ctx.RecordStepStart("s1")
 	time.Sleep(5 * time.Millisecond)
-	ctx.RecordStepSuccess("s1", map[string]any{"order": 1})
+	ctx.RecordStepSuccess("s1", map[string]any{"order": 1}, nil)
 
 	time.Sleep(5 * time.Millisecond)
 	ctx.RecordStepStart("s2")
 	time.Sleep(5 * time.Millisecond)
-	ctx.RecordStepSuccess("s2", map[string]any{"order": 2})
+	ctx.RecordStepSuccess("s2", map[string]any{"order": 2}, nil)
 
 	// Should return latest (s2)
 	output := ctx.GetLastStepOutput()

pkg/vmcp/composer/workflow_context.go

@@ -10,6 +10,8 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+
+	"github.com/stacklok/toolhive/pkg/vmcp"
 )
 
 // workflowContextManager manages workflow execution contexts.
@@ -87,13 +89,15 @@ func (ctx *WorkflowContext) RecordStepStart(stepID string) {
 
 // RecordStepSuccess records a successful step completion.
 // Thread-safe for concurrent step execution.
-func (ctx *WorkflowContext) RecordStepSuccess(stepID string, output map[string]any) {
+// The content parameter is optional (may be nil for non-tool steps like elicitation).
+func (ctx *WorkflowContext) RecordStepSuccess(stepID string, output map[string]any, content []vmcp.Content) {
 	ctx.mu.Lock()
 	defer ctx.mu.Unlock()
 
 	if result, exists := ctx.Steps[stepID]; exists {
 		result.Status = StepStatusCompleted
 		result.Output = output
+		result.Content = content
 		result.EndTime = time.Now()
 		result.Duration = result.EndTime.Sub(result.StartTime)
 	}
@@ -207,10 +211,16 @@ func (ctx *WorkflowContext) Clone() *WorkflowContext {
 
 	// Clone step results
 	for stepID, result := range ctx.Steps {
+		var contentCopy []vmcp.Content
+		if result.Content != nil {
+			contentCopy = make([]vmcp.Content, len(result.Content))
+			copy(contentCopy, result.Content)
+		}
 		clone.Steps[stepID] = &StepResult{
 			StepID:     result.StepID,
 			Status:     result.Status,
 			Output:     cloneMap(result.Output),
+			Content:    contentCopy,
 			Error:      result.Error,
 			StartTime:  result.StartTime,
 			EndTime:    result.EndTime,

pkg/vmcp/composer/workflow_engine.go

@@ -428,24 +428,32 @@ func (e *workflowEngine) executeToolStep(
 	}
 
 	// Call tool with retry logic
-	output, retryCount, err := e.callToolWithRetry(ctx, target, step, expandedArgs, workflowCtx)
+	result, retryCount, err := e.callToolWithRetry(ctx, target, step, expandedArgs, workflowCtx)
 
 	// Handle result
 	if err != nil {
 		return e.handleToolStepFailure(step, workflowCtx, retryCount, err)
 	}
 
-	return e.handleToolStepSuccess(ctx, step, workflowCtx, output, retryCount)
+	// Extract output map from result.
+	// Prefer StructuredContent (already a map), fall back to Content array conversion.
+	output := result.StructuredContent
+	if output == nil {
+		output = conversion.ContentArrayToMap(result.Content)
+	}
+
+	return e.handleToolStepSuccess(ctx, step, workflowCtx, output, result.Content, retryCount)
 }
 
 // callToolWithRetry calls a tool with retry logic using exponential backoff.
+// Returns the full ToolCallResult so callers can access both StructuredContent and Content.
 func (e *workflowEngine) callToolWithRetry(
 	ctx context.Context,
 	target *vmcp.BackendTarget,
 	step *WorkflowStep,
 	args map[string]any,
 	_ *WorkflowContext,
-) (map[string]any, int, error) {
+) (*vmcp.ToolCallResult, int, error) {
 	maxRetries, initialDelay := e.getRetryConfig(step)
 
 	// Configure exponential backoff
@@ -455,7 +463,7 @@ func (e *workflowEngine) callToolWithRetry(
 	expBackoff.Reset()
 
 	attemptCount := 0
-	operation := func() (map[string]any, error) {
+	operation := func() (*vmcp.ToolCallResult, error) {
 		attemptCount++
 		// TODO: For composite tools, we may want to propagate metadata from the parent request
 		result, err := e.backendClient.CallTool(ctx, target, step.Tool, args, nil)
@@ -482,30 +490,20 @@ func (e *workflowEngine) callToolWithRetry(
 			return nil, fmt.Errorf("%w: %s", vmcp.ErrToolExecutionFailed, errorMsg)
 		}
 
-		// Extract output map from result.
-		// Workflow logic uses map[string]any for template variable substitution.
-		// Prefer StructuredContent (already a map), fall back to Content array conversion.
-		// The _meta field is not needed for workflow execution, so we don't extract it.
-		if result.StructuredContent != nil {
-			return result.StructuredContent, nil
-		}
-
-		// Fallback: convert Content array to map for backward compatibility.
-		// This happens when backends return only Content without StructuredContent.
-		return conversion.ContentArrayToMap(result.Content), nil
+		return result, nil
 	}
 
 	// Execute with retry
 	// Safe conversion: maxRetries is capped by maxRetryCount constant (10)
-	output, err := backoff.Retry(ctx, operation,
+	result, err := backoff.Retry(ctx, operation,
 		backoff.WithBackOff(expBackoff),
 		backoff.WithMaxTries(uint(maxRetries+1)), // #nosec G115 -- +1 because it includes the initial attempt
 		backoff.WithNotify(func(_ error, duration time.Duration) {
 			slog.Debug("retrying step", "step", step.ID, "after", duration)
 		}),
 	)
 
-	return output, attemptCount - 1, err // Return retry count (attempts - 1)
+	return result, attemptCount - 1, err // Return retry count (attempts - 1)
 }
 
 // extractErrorMessage extracts a user-friendly error message from a failed tool call result.
@@ -595,9 +593,10 @@ func (e *workflowEngine) handleToolStepSuccess(
 	step *WorkflowStep,
 	workflowCtx *WorkflowContext,
 	output map[string]any,
+	content []vmcp.Content,
 	retryCount int,
 ) error {
-	workflowCtx.RecordStepSuccess(step.ID, output)
+	workflowCtx.RecordStepSuccess(step.ID, output, content)
 
 	// Update retry count
 	if result, exists := workflowCtx.GetStepResult(step.ID); exists {
@@ -698,7 +697,7 @@ func (*workflowEngine) handleElicitationAccept(
 		"content": response.Content,
 	}
 
-	workflowCtx.RecordStepSuccess(step.ID, output)
+	workflowCtx.RecordStepSuccess(step.ID, output, nil)
 	slog.Debug("step completed with user-provided data", "step", step.ID)
 	return nil
 }
@@ -772,7 +771,7 @@ func (*workflowEngine) handleElicitationAction(
 			"action":  reason,
 			"skipped": true,
 		}
-		workflowCtx.RecordStepSuccess(step.ID, output)
+		workflowCtx.RecordStepSuccess(step.ID, output, nil)
 		// Return a special error that the workflow engine can detect
 		// For now, we'll just complete the step successfully
 		return nil
@@ -795,7 +794,7 @@ func (*workflowEngine) handleElicitationAction(
 		output := map[string]any{
 			"action": reason,
 		}
-		workflowCtx.RecordStepSuccess(step.ID, output)
+		workflowCtx.RecordStepSuccess(step.ID, output, nil)
 		return nil
 
 	default:

pkg/vmcp/composer/workflow_engine_test.go

@@ -846,3 +846,64 @@ func TestWorkflowEngine_SessionEngine_ToolNotInList_ReturnsNilSchema(t *testing.
 	require.NoError(t, err)
 	assert.Equal(t, WorkflowStatusCompleted, result.Status)
 }
+
+func TestWorkflowEngine_EmbeddedResourceAccessibleFromTemplate(t *testing.T) {
+	t.Parallel()
+	te := newTestEngine(t)
+
+	// Step 1: tool returns structuredContent (schema-conformant) + embedded resource in Content array.
+	// Step 2: accesses structuredContent via .output and content array via .content.
+	// This keeps structuredContent clean for outputSchema validation while making
+	// content array data (text, resources) accessible through a separate namespace.
+	def := simpleWorkflow("resource-chain",
+		toolStep("fetch", "registry.get_referrer_content", map[string]any{
+			"image": "ghcr.io/org/repo:latest",
+		}),
+		toolStepWithDeps("analyze", "sbom.analyze", map[string]any{
+			"sbom_data": "{{.steps.fetch.content.resource}}",
+			"format":    "{{.steps.fetch.output.format}}",
+		}, []string{"fetch"}),
+	)
+
+	target := &vmcp.BackendTarget{
+		WorkloadID:   "test-backend",
+		WorkloadName: "test",
+		BaseURL:      "http://test:8080",
+	}
+	te.Router.EXPECT().RouteTool(gomock.Any(), "registry.get_referrer_content").Return(target, nil)
+	te.Backend.EXPECT().CallTool(gomock.Any(), target, "registry.get_referrer_content",
+		map[string]any{"image": "ghcr.io/org/repo:latest"}, gomock.Any()).
+		Return(&vmcp.ToolCallResult{
+			StructuredContent: map[string]any{
+				"contentType": "sbom",
+				"format":      "spdx",
+				"size":        float64(5347),
+			},
+			Content: []vmcp.Content{
+				{Type: vmcp.ContentTypeText, Text: "summary of SBOM"},
+				{Type: vmcp.ContentTypeResource, Text: `{"spdxVersion":"SPDX-2.3","name":"mypackage"}`, URI: "file://sbom.json"},
+			},
+		}, nil)
+
+	// Step 2: verify the template-expanded args pull from the right namespaces.
+	te.Router.EXPECT().RouteTool(gomock.Any(), "sbom.analyze").Return(target, nil)
+	te.Backend.EXPECT().CallTool(gomock.Any(), target, "sbom.analyze", gomock.Any(), gomock.Any()).
+		DoAndReturn(func(_ context.Context, _ *vmcp.BackendTarget, _ string, args map[string]any, _ map[string]any) (*vmcp.ToolCallResult, error) {
+			// .content.resource comes from the Content array's embedded resource
+			assert.Equal(t, `{"spdxVersion":"SPDX-2.3","name":"mypackage"}`, args["sbom_data"])
+			// .output.format comes from structuredContent
+			assert.Equal(t, "spdx", args["format"])
+			return &vmcp.ToolCallResult{
+				StructuredContent: map[string]any{"result": "analyzed"},
+				Content:           []vmcp.Content{},
+			}, nil
+		})
+
+	result, err := execute(t, te.Engine, def, nil)
+
+	require.NoError(t, err)
+	assert.Equal(t, WorkflowStatusCompleted, result.Status)
+	assert.Len(t, result.Steps, 2)
+	assert.Equal(t, StepStatusCompleted, result.Steps["fetch"].Status)
+	assert.Equal(t, StepStatusCompleted, result.Steps["analyze"].Status)
+}