SLVS-3097 Run extension E2E test on master and on PRs that change it

.github/workflows/build.yml

@@ -8,6 +8,12 @@ on:
       - branch-*
   pull_request:
   workflow_dispatch:
+    inputs:
+      extension-e2e:
+        description: Run the extension E2E test after this build (select the PR branch)
+        required: false
+        default: false
+        type: boolean
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -19,15 +25,38 @@ permissions:
   checks: write
 
 jobs:
+  extension-e2e:
+    needs: build
+    if: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/master') || (github.event_name == 'workflow_dispatch' && inputs.extension-e2e) || needs.build.outputs.extension-e2e-changed == 'true' }}
+    permissions:
+      contents: read
+      id-token: write
+      actions: read
+    uses: ./.github/workflows/extension-e2e.yml
+
   build:
     runs-on: warp-custom-sonarlint-visualstudio
     name: Build
     environment: ${{ (github.ref_name == 'master' || startsWith(github.ref_name, 'branch-')) && 'release-ready-build' || '' }}
+    outputs:
+      extension-e2e-changed: ${{ steps.extension-e2e-changes.outputs.changed }}
     steps:
       - uses: actions/checkout@v5
         with:
           fetch-depth: "0"
 
+      - name: Detect extension E2E changes
+        id: extension-e2e-changes
+        if: github.event_name == 'pull_request'
+        shell: bash
+        run: |
+          # HEAD is the PR merge commit; its first parent is the base branch tip.
+          if git diff --name-only HEAD^1 HEAD | grep -qE '^(\.github/workflows/extension-e2e\.yml$|tests/e2e/)'; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          fi
+
       - name: Fetch vault secrets
         id: secrets
         uses: SonarSource/vault-action-wrapper@v3
@@ -102,10 +131,10 @@ jobs:
         shell: bash
         id: branch-name
         run: |
-          if [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_OUTPUT
+          if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
+            echo "BRANCH_NAME=${GITHUB_HEAD_REF}" >> "$GITHUB_OUTPUT"
           else
-            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT
+            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> "$GITHUB_OUTPUT"
           fi
 
       - name: Prepare Build

.github/workflows/extension-e2e.yml

@@ -0,0 +1,68 @@
+name: Extension E2E test
+
+on:
+  workflow_call:
+
+permissions:
+  contents: read
+  id-token: write
+  actions: read
+
+jobs:
+  extension-e2e:
+    name: Extension E2E (VS 18)
+    runs-on: warp-custom-sonarlint-visualstudio
+    timeout-minutes: 20
+    env:
+      E2E_EXPECTED_VS_VERSION: '18.12.12405.243'
+      E2E_EXPECTED_VS_PRODUCT: Microsoft.VisualStudio.Product.Community
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Download this build's signed binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: binaries
+          path: ${{ runner.temp }}/extension-e2e-package
+
+      - name: Read artifact credentials for NuGet restore
+        id: secrets
+        uses: SonarSource/vault-action-wrapper@v3
+        with:
+          secrets: |
+            development/artifactory/token/{REPO_OWNER_NAME_DASH}-private-reader username | ARTIFACTORY_USER;
+            development/artifactory/token/{REPO_OWNER_NAME_DASH}-private-reader access_token | ARTIFACTORY_ACCESS_TOKEN;
+
+      - name: Build C# E2E tests
+        shell: powershell
+        env:
+          ARTIFACTORY_USER: ${{ fromJSON(steps.secrets.outputs.vault).ARTIFACTORY_USER }}
+          ARTIFACTORY_PASSWORD: ${{ fromJSON(steps.secrets.outputs.vault).ARTIFACTORY_ACCESS_TOKEN }}
+        run: |
+          $vswhere = "${env:ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere.exe"
+          $expectedVersion = [version]$env:E2E_EXPECTED_VS_VERSION
+          $versionRange = "[$($expectedVersion.Major).$($expectedVersion.Minor),$($expectedVersion.Major).$($expectedVersion.Minor + 1))"
+          $idePath = & $vswhere -latest -products $env:E2E_EXPECTED_VS_PRODUCT -version $versionRange -property installationPath
+          if ($LASTEXITCODE -ne 0 -or -not $idePath) { throw 'Full Visual Studio not found.' }
+          dotnet build tests/e2e/Extension.E2E.Tests.csproj -c Release -p:VsTargetVersion=2022 "-p:VisualStudioInstallPath=$idePath" -p:RestoreLockedMode=true
+          if ($LASTEXITCODE -ne 0) { throw 'C# E2E test build failed.' }
+
+      - name: Exercise one fresh installation
+        shell: powershell
+        run: |
+          $packages = @(Get-ChildItem -LiteralPath (Join-Path $env:RUNNER_TEMP 'extension-e2e-package') -Filter 'SonarLint.VSIX-*-2022.vsix' -File)
+          if ($packages.Count -ne 1) { throw "Expected exactly one VSIX from this build; found $($packages.Count)." }
+          $results = Join-Path $env:RUNNER_TEMP 'extension-e2e/run-1'
+          $env:E2E_VSIX_PATH = $packages[0].FullName
+          $env:E2E_RESULTS_DIRECTORY = Join-Path $results 'evidence'
+          dotnet test tests/e2e/Extension.E2E.Tests.csproj -c Release --no-build --no-restore --filter TestCategory=ExtensionE2E --logger 'trx;LogFileName=e2e.trx' --results-directory $results -p:VsTargetVersion=2022 -- RunConfiguration.TreatNoTestsAsError=true
+          if ($LASTEXITCODE -ne 0) { throw 'Extension E2E test failed.' }
+
+      - name: Upload E2E test evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: extension-e2e-vs${{ env.E2E_EXPECTED_VS_VERSION }}
+          path: ${{ runner.temp }}/extension-e2e/
+          if-no-files-found: warn
+          retention-days: 7

tests/e2e/ComMessageFilter.cs

@@ -0,0 +1,101 @@
+/*
+ * SonarLint for Visual Studio
+ * Copyright (C) SonarSource Sàrl
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+using System;
+using System.Runtime.InteropServices;
+using System.Threading;
+
+namespace SonarLint.VisualStudio.E2E
+{
+    [ComVisible(true)]
+    [ClassInterface(ClassInterfaceType.None)]
+    public sealed class ComMessageFilter : IOleMessageFilter, IDisposable
+    {
+        private const int ServerCallRetryLater = 2;
+        private const int RetryDelayMilliseconds = 250;
+        private const uint RetryTimeoutMilliseconds = 30000;
+        private IOleMessageFilter previousFilter;
+        private int registeredThreadId;
+
+        public int BusyCallCount { get; private set; }
+
+        public static ComMessageFilter Register()
+        {
+            if (Thread.CurrentThread.GetApartmentState() != ApartmentState.STA)
+            {
+                throw new InvalidOperationException("The COM message filter requires an STA thread.");
+            }
+            var filter = new ComMessageFilter();
+            var result = CoRegisterMessageFilter(filter, out filter.previousFilter);
+            if (result != 0)
+            {
+                throw new InvalidOperationException($"Could not register the COM message filter (HRESULT 0x{result:X8}).");
+            }
+            filter.registeredThreadId = Thread.CurrentThread.ManagedThreadId;
+            return filter;
+        }
+
+        public int HandleInComingCall(int callType, IntPtr taskCaller, int tickCount, IntPtr interfaceInfo) => 0;
+
+        public int RetryRejectedCall(IntPtr taskCallee, uint elapsedMilliseconds, int rejectType)
+        {
+            if (rejectType != ServerCallRetryLater) { return -1; }
+            BusyCallCount++;
+            // Do not retry permanent rejections or wait indefinitely for a busy IDE.
+            return elapsedMilliseconds < RetryTimeoutMilliseconds ? RetryDelayMilliseconds : -1;
+        }
+
+        public int MessagePending(IntPtr taskCallee, int tickCount, int pendingType) => 2;
+
+        public void Dispose()
+        {
+            if (registeredThreadId == 0) { return; }
+            if (registeredThreadId != Thread.CurrentThread.ManagedThreadId)
+            {
+                throw new InvalidOperationException("Restore the COM message filter on the thread that registered it.");
+            }
+            var result = CoRegisterMessageFilter(previousFilter, out _);
+            if (result != 0)
+            {
+                throw new InvalidOperationException($"Could not restore the COM message filter (HRESULT 0x{result:X8}).");
+            }
+            registeredThreadId = 0;
+            previousFilter = null;
+        }
+
+        [DllImport("ole32.dll", ExactSpelling = true)]
+        private static extern int CoRegisterMessageFilter(IOleMessageFilter newFilter, out IOleMessageFilter oldFilter);
+    }
+
+    [ComImport]
+    [Guid("00000016-0000-0000-C000-000000000046")]
+    [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
+    public interface IOleMessageFilter
+    {
+        [PreserveSig]
+        int HandleInComingCall(int callType, IntPtr taskCaller, int tickCount, IntPtr interfaceInfo);
+
+        [PreserveSig]
+        int RetryRejectedCall(IntPtr taskCallee, uint elapsedMilliseconds, int rejectType);
+
+        [PreserveSig]
+        int MessagePending(IntPtr taskCallee, int tickCount, int pendingType);
+    }
+}

tests/e2e/Extension.E2E.Tests.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net48</TargetFramework>
+    <IsTestProject>true</IsTestProject>
+    <RequiresSigning>false</RequiresSigning>
+    <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
+    <GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
+    <VisualStudioInteropPath Condition="'$(VisualStudioInteropPath)' == ''">$(VisualStudioInstallPath)\Common7\IDE\PublicAssemblies\Microsoft.VisualStudio.Interop.dll</VisualStudioInteropPath>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="MSTest" Version="4.4.1" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <Reference Include="Microsoft.VisualStudio.Interop" HintPath="$(VisualStudioInteropPath)" Condition="Exists('$(VisualStudioInteropPath)')" />
+    <Reference Include="EnvDTE" HintPath="$(VisualStudioInstallPath)\Common7\IDE\PublicAssemblies\envdte.dll" Condition="!Exists('$(VisualStudioInteropPath)')" />
+    <Reference Include="EnvDTE80" HintPath="$(VisualStudioInstallPath)\Common7\IDE\PublicAssemblies\envdte80.dll" Condition="!Exists('$(VisualStudioInteropPath)')" />
+    <Reference Include="System.IO.Compression" />
+    <Reference Include="System.IO.Compression.FileSystem" />
+    <Compile Remove="Fixture/**/*.cs" />
+    <None Include="Fixture/**/*" CopyToOutputDirectory="PreserveNewest" />
+  </ItemGroup>
+  <Target Name="CheckVisualStudioInterop" BeforeTargets="ResolveReferences">
+    <Error Condition="!Exists('$(VisualStudioInteropPath)') and (!Exists('$(VisualStudioInstallPath)\Common7\IDE\PublicAssemblies\envdte.dll') or !Exists('$(VisualStudioInstallPath)\Common7\IDE\PublicAssemblies\envdte80.dll'))" Text="Set VisualStudioInstallPath to the full Visual Studio installation." />
+  </Target>
+</Project>

tests/e2e/ExtensionE2ETests.cs

@@ -0,0 +1,67 @@
+/*
+ * SonarLint for Visual Studio
+ * Copyright (C) SonarSource Sàrl
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+using System;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace SonarLint.VisualStudio.E2E
+{
+    [TestClass]
+    [DoNotParallelize]
+    public class ExtensionE2ETests
+    {
+        public TestContext TestContext { get; set; }
+
+        [STATestMethod]
+        [TestCategory("ExtensionE2E")]
+        public void InstalledExtension_DetectsIssue_AndClearsItAfterFix()
+        {
+            using var messageFilter = ComMessageFilter.Register();
+            using var ide = new VisualStudioSession(
+                Environment.GetEnvironmentVariable("E2E_VSIX_PATH"),
+                Environment.GetEnvironmentVariable("E2E_RESULTS_DIRECTORY"),
+                TestContext,
+                messageFilter);
+            try
+            {
+                ide.InstallAndLaunch();
+                ide.OpenSample();
+
+                var finding = ide.WaitForFinding();
+                Assert.AreEqual(5, finding.Line);
+                Assert.AreEqual("Remove the unused local variable 'e2eUnused'.", finding.Description);
+                ide.SaveErrorList("errors-before-fix.json");
+
+                ide.RemoveUnusedVariableAndSave();
+
+                ide.WaitForFindingToDisappear(TimeSpan.FromSeconds(5));
+                Assert.AreEqual(0, ide.MatchingFindings().Length);
+                Assert.AreEqual(1, ide.RemainingFindings().Length, "The unedited Sonar finding must remain visible.");
+                ide.SaveErrorList("errors-after-fix.json");
+                ide.MarkPassed();
+            }
+            catch (Exception exception)
+            {
+                ide.RecordFailure(exception);
+                throw;
+            }
+        }
+    }
+}

tests/e2e/Fixture/Sample.cs.txt

@@ -0,0 +1,9 @@
+public class Sample
+{
+    public int Run()
+    {
+        int e2eUnused = 42;
+        int e2eRemaining = 43;
+        return 0;
+    }
+}

tests/e2e/Fixture/Sample.csproj

@@ -0,0 +1,10 @@
+<Project ToolsVersion="15.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <ProjectGuid>{657836E3-1079-4BFB-BA79-974702074374}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
+    <OutputPath>bin\</OutputPath>
+  </PropertyGroup>
+  <ItemGroup><Compile Include="Sample.cs" /></ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+</Project>

tests/e2e/Fixture/Sample.sln

@@ -0,0 +1,13 @@
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Sample", "Sample.csproj", "{657836E3-1079-4BFB-BA79-974702074374}"
+EndProject
+Global
+    GlobalSection(SolutionConfigurationPlatforms) = preSolution
+        Debug|Any CPU = Debug|Any CPU
+    EndGlobalSection
+    GlobalSection(ProjectConfigurationPlatforms) = postSolution
+        {657836E3-1079-4BFB-BA79-974702074374}.Debug|Any CPU.ActiveCfg = Debug|AnyCPU
+        {657836E3-1079-4BFB-BA79-974702074374}.Debug|Any CPU.Build.0 = Debug|AnyCPU
+    EndGlobalSection
+EndGlobal