ci: verify examples and Copilot integration tests

.github/workflows/live-examples.yml

@@ -0,0 +1,99 @@
+name: Live examples
+
+on:
+  schedule:
+    - cron: "17 9 * * 1-5"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  copilot-requests: write
+
+concurrency:
+  group: live-examples
+  cancel-in-progress: false
+
+jobs:
+  verify:
+    name: Verify examples and Copilot provider
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: go.sum
+
+      - name: Set up Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: 22
+
+      - name: Install Copilot CLI
+        shell: bash
+        run: npm install --global @github/copilot@1.0.75
+
+      - name: Verify examples
+        id: examples
+        continue-on-error: true
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          set +e
+          go run ./cmd/verifyexamples \
+            --parallel 4 \
+            --log live-examples.log \
+            --md live-examples.md
+          status=$?
+          set -e
+
+          if [[ -f live-examples.md ]]; then
+            cat live-examples.md >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "The example verifier exited before it could write a report." >> "$GITHUB_STEP_SUMMARY"
+          fi
+          exit "$status"
+
+      - name: Run Copilot integration tests
+        id: copilot_integration
+        if: ${{ !cancelled() }}
+        continue-on-error: true
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          set -o pipefail
+          go test -count=1 -v ./provider/copilotprovider -run '^TestE2E_' \
+            2>&1 | tee copilot-integration-tests.log
+
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: live-example-results
+          path: |
+            live-examples.log
+            live-examples.md
+            copilot-integration-tests.log
+          if-no-files-found: warn
+          retention-days: 14
+
+      - name: Check results
+        if: always()
+        shell: bash
+        env:
+          EXAMPLES_OUTCOME: ${{ steps.examples.outcome }}
+          COPILOT_INTEGRATION_OUTCOME: ${{ steps.copilot_integration.outcome }}
+        run: |
+          if [[ "$EXAMPLES_OUTCOME" != "success" || "$COPILOT_INTEGRATION_OUTCOME" != "success" ]]; then
+            echo "Example verification or Copilot integration tests failed."
+            exit 1
+          fi

cmd/verifyexamples/examples.go

@@ -708,14 +708,13 @@ var agentsExamples = []ExampleDefinition{
 		SkipReason:                   "Spawns a local stdio MCP server subprocess (go run) that requires live Foundry credentials.",
 	},
 	{
-		Name:        "02_agents_providers_github_copilot",
-		ProjectPath: "examples/02-agents/providers/github-copilot",
-		Inputs:      inputLines("Y", "Y", "Y"),
-		InputDelay:  3 * time.Second,
-		ExpectedOutputDescription: []string{
-			"The output should contain a response listing files in the current directory.",
-			"The output should not contain error messages or stack traces.",
-		},
+		Name:            "02_agents_providers_github_copilot",
+		ProjectPath:     "examples/02-agents/providers/github-copilot",
+		IsDeterministic: true,
+		Inputs:          inputLines("Y", "Y", "Y"),
+		InputDelay:      3 * time.Second,
+		MustContain:     []string{"main.go"},
+		MustNotContain:  []string{"panic:", "Permission denied"},
 	},
 	{
 		Name:                         "02_agents_skills_step01_file_based_skills",
@@ -927,19 +926,16 @@ var workflowExamples = []ExampleDefinition{
 		},
 	},
 	{
-		Name:        "03_workflows_checkpoint_checkpoint_with_human_in_the_loop",
-		ProjectPath: "examples/03-workflows/checkpoint/checkpoint_with_human_in_the_loop",
-		Inputs:      inputLines("50", "25", "40", "45", "42", "50", "25", "40", "45", "42"),
-		InputDelay:  time.Second,
+		Name:            "03_workflows_checkpoint_checkpoint_with_human_in_the_loop",
+		ProjectPath:     "examples/03-workflows/checkpoint/checkpoint_with_human_in_the_loop",
+		IsDeterministic: true,
+		Inputs:          inputLines("50", "25", "40", "45", "42", "50", "25", "40", "45", "42"),
+		InputDelay:      time.Second,
 		MustContain: []string{
 			"Workflow completed with result:",
 			"Number of checkpoints created:",
 			"Restored run completed with result:",
 		},
-		ExpectedOutputDescription: []string{
-			"The output should show a number guessing workflow with higher/lower hints that eventually reaches the correct number.",
-			"The output should demonstrate checkpoint save and restore behavior.",
-		},
 	},
 	{
 		Name:        "03_workflows_concurrent_map_reduce",
@@ -992,14 +988,12 @@ var workflowExamples = []ExampleDefinition{
 		MustContain:     []string{"[intermediate] normalized hello workflow", "[final] Summary of"},
 	},
 	{
-		Name:        "03_workflows_human_in_the_loop_human_in_the_loop_basic",
-		ProjectPath: "examples/03-workflows/human-in-the-loop/human_in_the_loop_basic",
-		Inputs:      inputLines("50", "25", "40", "45", "42"),
-		InputDelay:  time.Second,
-		MustContain: []string{"found in"},
-		ExpectedOutputDescription: []string{
-			"The output should show a number guessing game with higher/lower hints that eventually reaches the correct number 42.",
-		},
+		Name:            "03_workflows_human_in_the_loop_human_in_the_loop_basic",
+		ProjectPath:     "examples/03-workflows/human-in-the-loop/human_in_the_loop_basic",
+		IsDeterministic: true,
+		Inputs:          inputLines("50", "25", "40", "45", "42"),
+		InputDelay:      time.Second,
+		MustContain:     []string{"42 found in 5 tries!"},
 	},
 	{
 		Name:            "03_workflows_human_in_the_loop_imperative_request",

provider/copilotprovider/agent_e2e_test.go

@@ -0,0 +1,175 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+package copilotprovider_test
+
+import (
+	"context"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	copilot "github.com/github/copilot-sdk/go"
+	"github.com/microsoft/agent-framework-go/agent"
+	"github.com/microsoft/agent-framework-go/provider/copilotprovider"
+	"github.com/microsoft/agent-framework-go/tool"
+	"github.com/microsoft/agent-framework-go/tool/functool"
+)
+
+func TestE2E_RunTextReturnsResponse(t *testing.T) {
+	client := newE2EClient(t)
+	canaryAgent := copilotprovider.NewAgent(client, copilotprovider.AgentConfig{
+		SessionConfig: restrictedSessionConfig(nil),
+	})
+	session := newE2ESession(t, canaryAgent, client)
+
+	ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute)
+	defer cancel()
+	response, err := canaryAgent.RunText(
+		ctx,
+		"What is 2 + 2? Answer with just the number.",
+		agent.WithSession(session),
+		agent.Stream(false),
+	).Collect()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if response == nil || len(response.Messages) == 0 {
+		t.Fatal("response contains no messages")
+	}
+	if !strings.Contains(response.String(), "4") {
+		t.Fatalf("response = %q, want it to contain 4", response.String())
+	}
+}
+
+func TestE2E_RunTextStreamingReturnsUpdates(t *testing.T) {
+	client := newE2EClient(t)
+	canaryAgent := copilotprovider.NewAgent(client, copilotprovider.AgentConfig{
+		SessionConfig: restrictedSessionConfig(nil),
+	})
+	session := newE2ESession(t, canaryAgent, client)
+
+	ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute)
+	defer cancel()
+	var responseText strings.Builder
+	var updates int
+	for update, err := range canaryAgent.RunText(
+		ctx,
+		"What is 2 + 2? Answer with just the number.",
+		agent.WithSession(session),
+		agent.Stream(true),
+	) {
+		if err != nil {
+			t.Fatal(err)
+		}
+		updates++
+		responseText.WriteString(update.String())
+	}
+	if updates == 0 {
+		t.Fatal("stream contains no updates")
+	}
+	if !strings.Contains(responseText.String(), "4") {
+		t.Fatalf("streamed response = %q, want it to contain 4", responseText.String())
+	}
+}
+
+func TestE2E_RunTextInvokesFunctionTool(t *testing.T) {
+	client := newE2EClient(t)
+	var invoked atomic.Bool
+	weatherTool := functool.MustNew(functool.Config{
+		Name:        "get_weather",
+		Description: "Get the weather for a location",
+	}, func(_ context.Context, location string) (string, error) {
+		invoked.Store(true)
+		return "The weather in " + location + " is sunny with a high of 25C.", nil
+	})
+	config := restrictedSessionConfig([]string{"get_weather"})
+	config.OnPermissionRequest = copilot.PermissionHandler.ApproveAll
+	canaryAgent := copilotprovider.NewAgent(client, copilotprovider.AgentConfig{
+		SessionConfig: config,
+		Instructions:  "Always use the get_weather tool to answer weather questions.",
+		Config: agent.Config{
+			Tools: []tool.Tool{weatherTool},
+		},
+	})
+	session := newE2ESession(t, canaryAgent, client)
+
+	ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute)
+	defer cancel()
+	response, err := canaryAgent.RunText(
+		ctx,
+		"What is the weather like in Seattle?",
+		agent.WithSession(session),
+		agent.Stream(false),
+	).Collect()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if response == nil || len(response.Messages) == 0 {
+		t.Fatal("response contains no messages")
+	}
+	if !invoked.Load() {
+		t.Fatal("get_weather was not invoked")
+	}
+}
+
+func newE2EClient(t *testing.T) *copilot.Client {
+	t.Helper()
+	skipUnlessE2EEnabled(t)
+
+	options := &copilot.ClientOptions{}
+	if token := strings.TrimSpace(os.Getenv("GITHUB_TOKEN")); token != "" {
+		options.GitHubToken = token
+	}
+	client := copilot.NewClient(options)
+	ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+	defer cancel()
+	if err := client.Start(ctx); err != nil {
+		t.Fatalf("start Copilot client: %v", err)
+	}
+	t.Cleanup(client.ForceStop)
+	return client
+}
+
+func skipUnlessE2EEnabled(t *testing.T) {
+	t.Helper()
+	actionsAuth := strings.EqualFold(os.Getenv("GITHUB_ACTIONS"), "true") && strings.TrimSpace(os.Getenv("GITHUB_TOKEN")) != ""
+	localOptIn := strings.EqualFold(os.Getenv("RUN_COPILOT_INTEGRATION_TESTS"), "true")
+	if !actionsAuth && !localOptIn {
+		t.Skip("GitHub Actions auth is unavailable and RUN_COPILOT_INTEGRATION_TESTS is not true")
+	}
+}
+
+func restrictedSessionConfig(availableTools []string) *copilot.SessionConfig {
+	return &copilot.SessionConfig{
+		AvailableTools:                     availableTools,
+		EnableConfigDiscovery:              new(false),
+		EnableOnDemandInstructionDiscovery: new(false),
+		EnableFileHooks:                    new(false),
+		EnableHostGitOperations:            new(false),
+		EnableSessionStore:                 new(false),
+		EnableSkills:                       new(false),
+		InfiniteSessions: &copilot.InfiniteSessionConfig{
+			Enabled: new(false),
+		},
+	}
+}
+
+func newE2ESession(t *testing.T, canaryAgent *agent.Agent, client *copilot.Client) *agent.Session {
+	t.Helper()
+	session, err := canaryAgent.CreateSession(t.Context())
+	if err != nil {
+		t.Fatalf("create session: %v", err)
+	}
+	t.Cleanup(func() {
+		if sessionID := session.ServiceID(); sessionID != "" {
+			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+			defer cancel()
+			if err := client.DeleteSession(ctx, sessionID); err != nil {
+				t.Errorf("delete session: %v", err)
+			}
+		}
+	})
+	return session
+}