diff --git a/.github/workflows/live-examples.yml b/.github/workflows/live-examples.yml new file mode 100644 index 00000000..ac25a73e --- /dev/null +++ b/.github/workflows/live-examples.yml @@ -0,0 +1,99 @@ +name: Live examples + +on: + schedule: + - cron: "17 9 * * 1-5" + workflow_dispatch: + +permissions: + contents: read + copilot-requests: write + +concurrency: + group: live-examples + cancel-in-progress: false + +jobs: + verify: + name: Verify examples and Copilot provider + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache-dependency-path: go.sum + + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 22 + + - name: Install Copilot CLI + shell: bash + run: npm install --global @github/copilot@1.0.75 + + - name: Verify examples + id: examples + continue-on-error: true + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set +e + go run ./cmd/verifyexamples \ + --parallel 4 \ + --log live-examples.log \ + --md live-examples.md + status=$? + set -e + + if [[ -f live-examples.md ]]; then + cat live-examples.md >> "$GITHUB_STEP_SUMMARY" + else + echo "The example verifier exited before it could write a report." >> "$GITHUB_STEP_SUMMARY" + fi + exit "$status" + + - name: Run Copilot integration tests + id: copilot_integration + if: ${{ !cancelled() }} + continue-on-error: true + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -o pipefail + go test -count=1 -v ./provider/copilotprovider -run '^TestE2E_' \ + 2>&1 | tee copilot-integration-tests.log + + - name: Upload results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: live-example-results + path: | + live-examples.log + live-examples.md + copilot-integration-tests.log + if-no-files-found: warn + retention-days: 14 + + - name: Check results + if: always() + shell: bash + env: + EXAMPLES_OUTCOME: ${{ steps.examples.outcome }} + COPILOT_INTEGRATION_OUTCOME: ${{ steps.copilot_integration.outcome }} + run: | + if [[ "$EXAMPLES_OUTCOME" != "success" || "$COPILOT_INTEGRATION_OUTCOME" != "success" ]]; then + echo "Example verification or Copilot integration tests failed." + exit 1 + fi \ No newline at end of file diff --git a/cmd/verifyexamples/examples.go b/cmd/verifyexamples/examples.go index 437a6d2d..47abd96c 100644 --- a/cmd/verifyexamples/examples.go +++ b/cmd/verifyexamples/examples.go @@ -708,14 +708,13 @@ var agentsExamples = []ExampleDefinition{ SkipReason: "Spawns a local stdio MCP server subprocess (go run) that requires live Foundry credentials.", }, { - Name: "02_agents_providers_github_copilot", - ProjectPath: "examples/02-agents/providers/github-copilot", - Inputs: inputLines("Y", "Y", "Y"), - InputDelay: 3 * time.Second, - ExpectedOutputDescription: []string{ - "The output should contain a response listing files in the current directory.", - "The output should not contain error messages or stack traces.", - }, + Name: "02_agents_providers_github_copilot", + ProjectPath: "examples/02-agents/providers/github-copilot", + IsDeterministic: true, + Inputs: inputLines("Y", "Y", "Y"), + InputDelay: 3 * time.Second, + MustContain: []string{"main.go"}, + MustNotContain: []string{"panic:", "Permission denied"}, }, { Name: "02_agents_skills_step01_file_based_skills", @@ -927,19 +926,16 @@ var workflowExamples = []ExampleDefinition{ }, }, { - Name: "03_workflows_checkpoint_checkpoint_with_human_in_the_loop", - ProjectPath: "examples/03-workflows/checkpoint/checkpoint_with_human_in_the_loop", - Inputs: inputLines("50", "25", "40", "45", "42", "50", "25", "40", "45", "42"), - InputDelay: time.Second, + Name: "03_workflows_checkpoint_checkpoint_with_human_in_the_loop", + ProjectPath: "examples/03-workflows/checkpoint/checkpoint_with_human_in_the_loop", + IsDeterministic: true, + Inputs: inputLines("50", "25", "40", "45", "42", "50", "25", "40", "45", "42"), + InputDelay: time.Second, MustContain: []string{ "Workflow completed with result:", "Number of checkpoints created:", "Restored run completed with result:", }, - ExpectedOutputDescription: []string{ - "The output should show a number guessing workflow with higher/lower hints that eventually reaches the correct number.", - "The output should demonstrate checkpoint save and restore behavior.", - }, }, { Name: "03_workflows_concurrent_map_reduce", @@ -992,14 +988,12 @@ var workflowExamples = []ExampleDefinition{ MustContain: []string{"[intermediate] normalized hello workflow", "[final] Summary of"}, }, { - Name: "03_workflows_human_in_the_loop_human_in_the_loop_basic", - ProjectPath: "examples/03-workflows/human-in-the-loop/human_in_the_loop_basic", - Inputs: inputLines("50", "25", "40", "45", "42"), - InputDelay: time.Second, - MustContain: []string{"found in"}, - ExpectedOutputDescription: []string{ - "The output should show a number guessing game with higher/lower hints that eventually reaches the correct number 42.", - }, + Name: "03_workflows_human_in_the_loop_human_in_the_loop_basic", + ProjectPath: "examples/03-workflows/human-in-the-loop/human_in_the_loop_basic", + IsDeterministic: true, + Inputs: inputLines("50", "25", "40", "45", "42"), + InputDelay: time.Second, + MustContain: []string{"42 found in 5 tries!"}, }, { Name: "03_workflows_human_in_the_loop_imperative_request", diff --git a/provider/copilotprovider/agent_e2e_test.go b/provider/copilotprovider/agent_e2e_test.go new file mode 100644 index 00000000..71b5796e --- /dev/null +++ b/provider/copilotprovider/agent_e2e_test.go @@ -0,0 +1,175 @@ +// Copyright (c) Microsoft. All rights reserved. + +package copilotprovider_test + +import ( + "context" + "os" + "strings" + "sync/atomic" + "testing" + "time" + + copilot "github.com/github/copilot-sdk/go" + "github.com/microsoft/agent-framework-go/agent" + "github.com/microsoft/agent-framework-go/provider/copilotprovider" + "github.com/microsoft/agent-framework-go/tool" + "github.com/microsoft/agent-framework-go/tool/functool" +) + +func TestE2E_RunTextReturnsResponse(t *testing.T) { + client := newE2EClient(t) + canaryAgent := copilotprovider.NewAgent(client, copilotprovider.AgentConfig{ + SessionConfig: restrictedSessionConfig(nil), + }) + session := newE2ESession(t, canaryAgent, client) + + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute) + defer cancel() + response, err := canaryAgent.RunText( + ctx, + "What is 2 + 2? Answer with just the number.", + agent.WithSession(session), + agent.Stream(false), + ).Collect() + if err != nil { + t.Fatal(err) + } + if response == nil || len(response.Messages) == 0 { + t.Fatal("response contains no messages") + } + if !strings.Contains(response.String(), "4") { + t.Fatalf("response = %q, want it to contain 4", response.String()) + } +} + +func TestE2E_RunTextStreamingReturnsUpdates(t *testing.T) { + client := newE2EClient(t) + canaryAgent := copilotprovider.NewAgent(client, copilotprovider.AgentConfig{ + SessionConfig: restrictedSessionConfig(nil), + }) + session := newE2ESession(t, canaryAgent, client) + + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute) + defer cancel() + var responseText strings.Builder + var updates int + for update, err := range canaryAgent.RunText( + ctx, + "What is 2 + 2? Answer with just the number.", + agent.WithSession(session), + agent.Stream(true), + ) { + if err != nil { + t.Fatal(err) + } + updates++ + responseText.WriteString(update.String()) + } + if updates == 0 { + t.Fatal("stream contains no updates") + } + if !strings.Contains(responseText.String(), "4") { + t.Fatalf("streamed response = %q, want it to contain 4", responseText.String()) + } +} + +func TestE2E_RunTextInvokesFunctionTool(t *testing.T) { + client := newE2EClient(t) + var invoked atomic.Bool + weatherTool := functool.MustNew(functool.Config{ + Name: "get_weather", + Description: "Get the weather for a location", + }, func(_ context.Context, location string) (string, error) { + invoked.Store(true) + return "The weather in " + location + " is sunny with a high of 25C.", nil + }) + config := restrictedSessionConfig([]string{"get_weather"}) + config.OnPermissionRequest = copilot.PermissionHandler.ApproveAll + canaryAgent := copilotprovider.NewAgent(client, copilotprovider.AgentConfig{ + SessionConfig: config, + Instructions: "Always use the get_weather tool to answer weather questions.", + Config: agent.Config{ + Tools: []tool.Tool{weatherTool}, + }, + }) + session := newE2ESession(t, canaryAgent, client) + + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute) + defer cancel() + response, err := canaryAgent.RunText( + ctx, + "What is the weather like in Seattle?", + agent.WithSession(session), + agent.Stream(false), + ).Collect() + if err != nil { + t.Fatal(err) + } + if response == nil || len(response.Messages) == 0 { + t.Fatal("response contains no messages") + } + if !invoked.Load() { + t.Fatal("get_weather was not invoked") + } +} + +func newE2EClient(t *testing.T) *copilot.Client { + t.Helper() + skipUnlessE2EEnabled(t) + + options := &copilot.ClientOptions{} + if token := strings.TrimSpace(os.Getenv("GITHUB_TOKEN")); token != "" { + options.GitHubToken = token + } + client := copilot.NewClient(options) + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + if err := client.Start(ctx); err != nil { + t.Fatalf("start Copilot client: %v", err) + } + t.Cleanup(client.ForceStop) + return client +} + +func skipUnlessE2EEnabled(t *testing.T) { + t.Helper() + actionsAuth := strings.EqualFold(os.Getenv("GITHUB_ACTIONS"), "true") && strings.TrimSpace(os.Getenv("GITHUB_TOKEN")) != "" + localOptIn := strings.EqualFold(os.Getenv("RUN_COPILOT_INTEGRATION_TESTS"), "true") + if !actionsAuth && !localOptIn { + t.Skip("GitHub Actions auth is unavailable and RUN_COPILOT_INTEGRATION_TESTS is not true") + } +} + +func restrictedSessionConfig(availableTools []string) *copilot.SessionConfig { + return &copilot.SessionConfig{ + AvailableTools: availableTools, + EnableConfigDiscovery: new(false), + EnableOnDemandInstructionDiscovery: new(false), + EnableFileHooks: new(false), + EnableHostGitOperations: new(false), + EnableSessionStore: new(false), + EnableSkills: new(false), + InfiniteSessions: &copilot.InfiniteSessionConfig{ + Enabled: new(false), + }, + } +} + +func newE2ESession(t *testing.T, canaryAgent *agent.Agent, client *copilot.Client) *agent.Session { + t.Helper() + session, err := canaryAgent.CreateSession(t.Context()) + if err != nil { + t.Fatalf("create session: %v", err) + } + t.Cleanup(func() { + if sessionID := session.ServiceID(); sessionID != "" { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := client.DeleteSession(ctx, sessionID); err != nil { + t.Errorf("delete session: %v", err) + } + } + }) + return session +}