Expose content array as separate template namespace instead of merging

The initial approach merged content array keys flat into
structuredContent, which violates MCP outputSchema when
additionalProperties: false is set.

Instead, expose the content array as an independent template
namespace ({{.steps.X.content.*}}) alongside the existing output
namespace ({{.steps.X.output.*}}). This keeps structuredContent
clean for schema validation while making text and embedded
resource data accessible for workflow chaining.

Changes:
- Revert flat merge in client.go and mcp_session.go
- Add Content field to StepResult, wire through RecordStepSuccess
- Expose content key in buildStepsContext via ContentArrayToMap
- Fix Clone() to copy Content field
- Update tests for separate namespace approach

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: JAORMX

pkg/vmcp/client/client.go

@@ -657,19 +657,10 @@ func (h *httpBackendClient) CallTool(
 		}
 	}
 
-	if structuredContent != nil {
-		// Merge content array keys (text, resource, etc.) without overwriting structured keys.
-		// This makes both the tool's structured metadata and actual content accessible.
-		contentMap := conversion.ContentArrayToMap(contentArray)
-		for k, v := range contentMap {
-			if _, exists := structuredContent[k]; !exists {
-				structuredContent[k] = v
-			}
-		}
-	} else {
-		// No structured content: convert result contents to a map for backward compatibility.
-		// MCP tools return an array of Content interface (TextContent, ImageContent, etc.).
-		// Text content is stored under "text" key, accessible via {{.steps.stepID.output.text}}.
+	// If no structured content, convert result contents to a map for backward compatibility.
+	// MCP tools return an array of Content interface (TextContent, ImageContent, etc.).
+	// Text content is stored under "text" key, accessible via {{.steps.stepID.output.text}}.
+	if structuredContent == nil {
 		structuredContent = conversion.ContentArrayToMap(contentArray)
 	}
 

pkg/vmcp/composer/composer.go

@@ -13,6 +13,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/stacklok/toolhive/pkg/vmcp"
 	"github.com/stacklok/toolhive/pkg/vmcp/config"
 )
 
@@ -212,9 +213,14 @@ type StepResult struct {
 	// Status is the step status.
 	Status StepStatusType
 
-	// Output contains the step output data.
+	// Output contains the step output data (from StructuredContent or ContentArrayToMap fallback).
 	Output map[string]any
 
+	// Content holds the raw content array from the tool call result.
+	// This is exposed separately in templates via {{.steps.stepID.content.*}} so that
+	// structuredContent remains clean for outputSchema validation.
+	Content []vmcp.Content
+
 	// Error contains error information if the step failed.
 	Error error
 

pkg/vmcp/composer/template_expander.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/stacklok/toolhive/pkg/templates"
+	"github.com/stacklok/toolhive/pkg/vmcp/conversion"
 )
 
 const (
@@ -163,7 +164,10 @@ func (e *defaultTemplateExpander) expandString(
 }
 
 // buildStepsContext converts StepResult map to a template-friendly structure.
-// This provides access to step outputs via {{.steps.stepid.output.field}}.
+// This provides access to step outputs via:
+//   - {{.steps.stepid.output.field}} for structuredContent fields
+//   - {{.steps.stepid.content.text}} for text content from the content array
+//   - {{.steps.stepid.content.resource}} for embedded resource content from the content array
 func (*defaultTemplateExpander) buildStepsContext(workflowCtx *WorkflowContext) map[string]any {
 	// Acquire read lock to safely access Steps map during concurrent execution
 	workflowCtx.mu.RLock()
@@ -173,8 +177,9 @@ func (*defaultTemplateExpander) buildStepsContext(workflowCtx *WorkflowContext)
 
 	for stepID, result := range workflowCtx.Steps {
 		stepData := map[string]any{
-			"status": string(result.Status),
-			"output": result.Output,
+			"status":  string(result.Status),
+			"output":  result.Output,
+			"content": conversion.ContentArrayToMap(result.Content),
 		}
 
 		// Add error information if step failed

pkg/vmcp/composer/template_expander_test.go

@@ -10,6 +10,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/stacklok/toolhive/pkg/vmcp"
 )
 
 func TestTemplateExpander_Expand(t *testing.T) {
@@ -74,6 +76,40 @@ func TestTemplateExpander_Expand(t *testing.T) {
 			params:   map[string]any{},
 			expected: map[string]any{"val": "<no value>"},
 		},
+		{
+			name: "step content resource substitution",
+			data: map[string]any{"sbom": "{{.steps.fetch.content.resource}}"},
+			steps: map[string]*StepResult{
+				"fetch": {
+					Status: StepStatusCompleted,
+					Output: map[string]any{"format": "spdx"},
+					Content: []vmcp.Content{
+						{Type: vmcp.ContentTypeResource, Text: `{"spdxVersion":"SPDX-2.3"}`, URI: "file://sbom.json"},
+					},
+				},
+			},
+			expected: map[string]any{"sbom": `{"spdxVersion":"SPDX-2.3"}`},
+		},
+		{
+			name: "step output and content are independent namespaces",
+			data: map[string]any{
+				"format": "{{.steps.fetch.output.format}}",
+				"text":   "{{.steps.fetch.content.text}}",
+			},
+			steps: map[string]*StepResult{
+				"fetch": {
+					Status: StepStatusCompleted,
+					Output: map[string]any{"format": "spdx", "text": "structured text"},
+					Content: []vmcp.Content{
+						{Type: vmcp.ContentTypeText, Text: "content array text"},
+					},
+				},
+			},
+			expected: map[string]any{
+				"format": "spdx",
+				"text":   "content array text",
+			},
+		},
 	}
 
 	expander := NewTemplateExpander()
@@ -154,7 +190,7 @@ func TestWorkflowContext_Lifecycle(t *testing.T) {
 	assert.Equal(t, StepStatusRunning, ctx.Steps["s1"].Status)
 
 	time.Sleep(10 * time.Millisecond)
-	ctx.RecordStepSuccess("s1", map[string]any{"result": "ok"})
+	ctx.RecordStepSuccess("s1", map[string]any{"result": "ok"}, nil)
 	assert.Equal(t, StepStatusCompleted, ctx.Steps["s1"].Status)
 	assert.Greater(t, ctx.Steps["s1"].Duration, time.Duration(0))
 
@@ -185,12 +221,12 @@ func TestWorkflowContext_GetLastStepOutput(t *testing.T) {
 	// Add steps with different completion times
 	ctx.RecordStepStart("s1")
 	time.Sleep(5 * time.Millisecond)
-	ctx.RecordStepSuccess("s1", map[string]any{"order": 1})
+	ctx.RecordStepSuccess("s1", map[string]any{"order": 1}, nil)
 
 	time.Sleep(5 * time.Millisecond)
 	ctx.RecordStepStart("s2")
 	time.Sleep(5 * time.Millisecond)
-	ctx.RecordStepSuccess("s2", map[string]any{"order": 2})
+	ctx.RecordStepSuccess("s2", map[string]any{"order": 2}, nil)
 
 	// Should return latest (s2)
 	output := ctx.GetLastStepOutput()

pkg/vmcp/composer/workflow_context.go

@@ -10,6 +10,8 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+
+	"github.com/stacklok/toolhive/pkg/vmcp"
 )
 
 // workflowContextManager manages workflow execution contexts.
@@ -87,13 +89,15 @@ func (ctx *WorkflowContext) RecordStepStart(stepID string) {
 
 // RecordStepSuccess records a successful step completion.
 // Thread-safe for concurrent step execution.
-func (ctx *WorkflowContext) RecordStepSuccess(stepID string, output map[string]any) {
+// The content parameter is optional (may be nil for non-tool steps like elicitation).
+func (ctx *WorkflowContext) RecordStepSuccess(stepID string, output map[string]any, content []vmcp.Content) {
 	ctx.mu.Lock()
 	defer ctx.mu.Unlock()
 
 	if result, exists := ctx.Steps[stepID]; exists {
 		result.Status = StepStatusCompleted
 		result.Output = output
+		result.Content = content
 		result.EndTime = time.Now()
 		result.Duration = result.EndTime.Sub(result.StartTime)
 	}
@@ -207,10 +211,16 @@ func (ctx *WorkflowContext) Clone() *WorkflowContext {
 
 	// Clone step results
 	for stepID, result := range ctx.Steps {
+		var contentCopy []vmcp.Content
+		if result.Content != nil {
+			contentCopy = make([]vmcp.Content, len(result.Content))
+			copy(contentCopy, result.Content)
+		}
 		clone.Steps[stepID] = &StepResult{
 			StepID:     result.StepID,
 			Status:     result.Status,
 			Output:     cloneMap(result.Output),
+			Content:    contentCopy,
 			Error:      result.Error,
 			StartTime:  result.StartTime,
 			EndTime:    result.EndTime,

pkg/vmcp/composer/workflow_engine.go

@@ -428,24 +428,32 @@ func (e *workflowEngine) executeToolStep(
 	}
 
 	// Call tool with retry logic
-	output, retryCount, err := e.callToolWithRetry(ctx, target, step, expandedArgs, workflowCtx)
+	result, retryCount, err := e.callToolWithRetry(ctx, target, step, expandedArgs, workflowCtx)
 
 	// Handle result
 	if err != nil {
 		return e.handleToolStepFailure(step, workflowCtx, retryCount, err)
 	}
 
-	return e.handleToolStepSuccess(ctx, step, workflowCtx, output, retryCount)
+	// Extract output map from result.
+	// Prefer StructuredContent (already a map), fall back to Content array conversion.
+	output := result.StructuredContent
+	if output == nil {
+		output = conversion.ContentArrayToMap(result.Content)
+	}
+
+	return e.handleToolStepSuccess(ctx, step, workflowCtx, output, result.Content, retryCount)
 }
 
 // callToolWithRetry calls a tool with retry logic using exponential backoff.
+// Returns the full ToolCallResult so callers can access both StructuredContent and Content.
 func (e *workflowEngine) callToolWithRetry(
 	ctx context.Context,
 	target *vmcp.BackendTarget,
 	step *WorkflowStep,
 	args map[string]any,
 	_ *WorkflowContext,
-) (map[string]any, int, error) {
+) (*vmcp.ToolCallResult, int, error) {
 	maxRetries, initialDelay := e.getRetryConfig(step)
 
 	// Configure exponential backoff
@@ -455,7 +463,7 @@ func (e *workflowEngine) callToolWithRetry(
 	expBackoff.Reset()
 
 	attemptCount := 0
-	operation := func() (map[string]any, error) {
+	operation := func() (*vmcp.ToolCallResult, error) {
 		attemptCount++
 		// TODO: For composite tools, we may want to propagate metadata from the parent request
 		result, err := e.backendClient.CallTool(ctx, target, step.Tool, args, nil)
@@ -482,30 +490,20 @@ func (e *workflowEngine) callToolWithRetry(
 			return nil, fmt.Errorf("%w: %s", vmcp.ErrToolExecutionFailed, errorMsg)
 		}
 
-		// Extract output map from result.
-		// Workflow logic uses map[string]any for template variable substitution.
-		// Prefer StructuredContent (already a map), fall back to Content array conversion.
-		// The _meta field is not needed for workflow execution, so we don't extract it.
-		if result.StructuredContent != nil {
-			return result.StructuredContent, nil
-		}
-
-		// Fallback: convert Content array to map for backward compatibility.
-		// This happens when backends return only Content without StructuredContent.
-		return conversion.ContentArrayToMap(result.Content), nil
+		return result, nil
 	}
 
 	// Execute with retry
 	// Safe conversion: maxRetries is capped by maxRetryCount constant (10)
-	output, err := backoff.Retry(ctx, operation,
+	result, err := backoff.Retry(ctx, operation,
 		backoff.WithBackOff(expBackoff),
 		backoff.WithMaxTries(uint(maxRetries+1)), // #nosec G115 -- +1 because it includes the initial attempt
 		backoff.WithNotify(func(_ error, duration time.Duration) {
 			slog.Debug("retrying step", "step", step.ID, "after", duration)
 		}),
 	)
 
-	return output, attemptCount - 1, err // Return retry count (attempts - 1)
+	return result, attemptCount - 1, err // Return retry count (attempts - 1)
 }
 
 // extractErrorMessage extracts a user-friendly error message from a failed tool call result.
@@ -595,9 +593,10 @@ func (e *workflowEngine) handleToolStepSuccess(
 	step *WorkflowStep,
 	workflowCtx *WorkflowContext,
 	output map[string]any,
+	content []vmcp.Content,
 	retryCount int,
 ) error {
-	workflowCtx.RecordStepSuccess(step.ID, output)
+	workflowCtx.RecordStepSuccess(step.ID, output, content)
 
 	// Update retry count
 	if result, exists := workflowCtx.GetStepResult(step.ID); exists {
@@ -698,7 +697,7 @@ func (*workflowEngine) handleElicitationAccept(
 		"content": response.Content,
 	}
 
-	workflowCtx.RecordStepSuccess(step.ID, output)
+	workflowCtx.RecordStepSuccess(step.ID, output, nil)
 	slog.Debug("step completed with user-provided data", "step", step.ID)
 	return nil
 }
@@ -772,7 +771,7 @@ func (*workflowEngine) handleElicitationAction(
 			"action":  reason,
 			"skipped": true,
 		}
-		workflowCtx.RecordStepSuccess(step.ID, output)
+		workflowCtx.RecordStepSuccess(step.ID, output, nil)
 		// Return a special error that the workflow engine can detect
 		// For now, we'll just complete the step successfully
 		return nil
@@ -795,7 +794,7 @@ func (*workflowEngine) handleElicitationAction(
 		output := map[string]any{
 			"action": reason,
 		}
-		workflowCtx.RecordStepSuccess(step.ID, output)
+		workflowCtx.RecordStepSuccess(step.ID, output, nil)
 		return nil
 
 	default:

pkg/vmcp/composer/workflow_engine_test.go

@@ -851,20 +851,20 @@ func TestWorkflowEngine_EmbeddedResourceAccessibleFromTemplate(t *testing.T) {
 	t.Parallel()
 	te := newTestEngine(t)
 
-	// Step 1: tool returns structuredContent + embedded resource in Content array.
-	// The client layer merges content array keys into structuredContent.
-	// Step 2: uses {{.steps.fetch.output.resource}} from the merged result.
+	// Step 1: tool returns structuredContent (schema-conformant) + embedded resource in Content array.
+	// Step 2: accesses structuredContent via .output and content array via .content.
+	// This keeps structuredContent clean for outputSchema validation while making
+	// content array data (text, resources) accessible through a separate namespace.
 	def := simpleWorkflow("resource-chain",
 		toolStep("fetch", "registry.get_referrer_content", map[string]any{
 			"image": "ghcr.io/org/repo:latest",
 		}),
 		toolStepWithDeps("analyze", "sbom.analyze", map[string]any{
-			"sbom_data": "{{.steps.fetch.output.resource}}",
+			"sbom_data": "{{.steps.fetch.content.resource}}",
 			"format":    "{{.steps.fetch.output.format}}",
 		}, []string{"fetch"}),
 	)
 
-	// Step 1 returns merged structuredContent (as the real client would after the fix).
 	target := &vmcp.BackendTarget{
 		WorkloadID:   "test-backend",
 		WorkloadName: "test",
@@ -877,21 +877,21 @@ func TestWorkflowEngine_EmbeddedResourceAccessibleFromTemplate(t *testing.T) {
 			StructuredContent: map[string]any{
 				"contentType": "sbom",
 				"format":      "spdx",
-				"text":        "summary of SBOM",
-				"resource":    `{"spdxVersion":"SPDX-2.3","name":"mypackage"}`,
+				"size":        float64(5347),
 			},
 			Content: []vmcp.Content{
 				{Type: vmcp.ContentTypeText, Text: "summary of SBOM"},
 				{Type: vmcp.ContentTypeResource, Text: `{"spdxVersion":"SPDX-2.3","name":"mypackage"}`, URI: "file://sbom.json"},
 			},
 		}, nil)
 
-	// Step 2: verify that the template-expanded args contain the resource data.
+	// Step 2: verify the template-expanded args pull from the right namespaces.
 	te.Router.EXPECT().RouteTool(gomock.Any(), "sbom.analyze").Return(target, nil)
 	te.Backend.EXPECT().CallTool(gomock.Any(), target, "sbom.analyze", gomock.Any(), gomock.Any()).
 		DoAndReturn(func(_ context.Context, _ *vmcp.BackendTarget, _ string, args map[string]any, _ map[string]any) (*vmcp.ToolCallResult, error) {
-			// Verify the template expanded the resource content correctly
+			// .content.resource comes from the Content array's embedded resource
 			assert.Equal(t, `{"spdxVersion":"SPDX-2.3","name":"mypackage"}`, args["sbom_data"])
+			// .output.format comes from structuredContent
 			assert.Equal(t, "spdx", args["format"])
 			return &vmcp.ToolCallResult{
 				StructuredContent: map[string]any{"result": "analyzed"},