envoyproxy · mattklein123 · Apr 21, 2020 · Mar 25, 2020 · Mar 17, 2020 · Mar 26, 2020
diff --git a/.azure-pipelines/windows.yml b/.azure-pipelines/windows.yml
@@ -12,14 +12,19 @@ jobs:
   steps:
     - powershell: |
         .\ci\windows_ci_setup.ps1
+        Write-Host "##vso[task.prependpath]$env:TOOLS_BIN_DIR\usr\bin"
+        Write-Host "##vso[task.prependpath]$env:VC_TOOLS_BIN_X64;$env:VC_CMAKE_PATH\CMake\bin;$env:VC_CMAKE_PATH\Ninja"
         Write-Host "##vso[task.prependpath]$env:TOOLS_BIN_DIR"
       displayName: "Install dependencies"
       env:
         TOOLS_BIN_DIR: $(Pipeline.Workspace)\bin
+        VC_CMAKE_PATH: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\Common7\\IDE\\CommonExtensions\\Microsoft\\CMake"
+        VC_TOOLS_BIN_X64: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\VC\\Tools\\MSVC\\14.24.28314\\bin\\HostX64\\x64"
 
     - bash: ci/windows_ci_steps.sh
       displayName: "Run Windows CI"
       env:
-        TMPDIR: $(Agent.TempDirectory)
         BAZEL_VC: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\VC"
-        BAZEL_SH: "C:\\Program Files\\Git\\bin\\bash.exe"
+        BAZEL_SH: $(Pipeline.Workspace)/bin/usr/bin/bash.exe
+        MSYS2_ARG_CONV_EXCL: "*"
+        TMPDIR: $(Agent.TempDirectory)
diff --git a/bazel/external/kafka_int32.patch b/bazel/external/kafka_int32.patch
@@ -0,0 +1,27 @@
+--- DescribeGroupsResponse.json	2020-03-25 16:12:16.373302600 -0400
++++ DescribeGroupsResponse.json	2020-03-25 16:11:16.184156200 -0400
+@@ -63,7 +63,7 @@
+         { "name": "MemberAssignment", "type": "bytes", "versions": "0+",
+           "about": "The current assignment provided by the group leader." }
+       ]},
+-      { "name": "AuthorizedOperations", "type": "int32", "versions": "3+",  "default": "-2147483648",
++      { "name": "AuthorizedOperations", "type": "int32", "versions": "3+",  "default": "INT32_MIN",
+         "about": "32-bit bitfield to represent authorized operations for this group." }
+     ]}
+   ]
+
+--- MetadataResponse.json	2020-03-25 15:53:36.319161000 -0400
++++ MetadataResponse.json	2020-03-25 15:54:11.510400000 -0400
+@@ -81,10 +81,10 @@
+         { "name": "OfflineReplicas", "type": "[]int32", "versions": "5+", "ignorable": true,
+           "about": "The set of offline replicas of this partition." }
+       ]},
+-      { "name": "TopicAuthorizedOperations", "type": "int32", "versions": "8+", "default": "-2147483648",
++      { "name": "TopicAuthorizedOperations", "type": "int32", "versions": "8+", "default": "INT32_MIN",
+         "about": "32-bit bitfield to represent authorized operations for this topic." }
+     ]},
+-    { "name": "ClusterAuthorizedOperations", "type": "int32", "versions": "8+", "default": "-2147483648",
++    { "name": "ClusterAuthorizedOperations", "type": "int32", "versions": "8+", "default": "INT32_MIN",
+       "about": "32-bit bitfield to represent authorized operations for this cluster." }
+   ]
+ }
diff --git a/bazel/foreign_cc/BUILD b/bazel/foreign_cc/BUILD
@@ -36,8 +36,6 @@ cc_library(
     ],
 )
 
-# autotools packages are unusable on Windows as-is
-# TODO: Consider our own luajit.BUILD file as we do with many other packages
 configure_make(
     name = "luajit",
     configure_command = "build.py",
@@ -51,10 +49,10 @@ configure_make(
     lib_source = "@com_github_luajit_luajit//:all",
     make_commands = [],
     out_include_dir = "include/luajit-2.1",
-    static_libraries = [
-        "libluajit-5.1.a",
-    ],
-    tags = ["skip_on_windows"],
+    static_libraries = select({
+        "//bazel:windows_x86_64": ["lua51.lib"],
+        "//conditions:default": ["libluajit-5.1.a"],
+    }),
 )
 
 configure_make(
@@ -70,10 +68,10 @@ configure_make(
     lib_source = "@com_github_moonjit_moonjit//:all",
     make_commands = [],
     out_include_dir = "include/moonjit-2.2",
-    static_libraries = [
-        "libluajit-5.1.a",
-    ],
-    tags = ["skip_on_windows"],
+    static_libraries = select({
+        "//bazel:windows_x86_64": ["lua51.lib"],
+        "//conditions:default": ["libluajit-5.1.a"],
+    }),
 )
 
 envoy_cmake_external(

diff --git a/bazel/foreign_cc/luajit.patch b/bazel/foreign_cc/luajit.patch
@@ -48,7 +48,7 @@ new file mode 100755
 index 0000000..9c71271
 --- /dev/null
 +++ b/build.py
-@@ -0,0 +1,39 @@
+@@ -0,0 +1,56 @@
 +#!/usr/bin/env python
 +
 +import argparse
@@ -86,5 +86,22 @@ index 0000000..9c71271
 +
 +    os.system('make V=1 PREFIX="{}" install'.format(args.prefix))
 +
-+main()
++def win_main():
++    src_dir = os.path.dirname(os.path.realpath(__file__))
++    dst_dir = os.getcwd() + "/luajit"
++    shutil.copytree(src_dir, os.path.basename(src_dir))
++    os.chdir(os.path.basename(src_dir) + "/src")
++    os.system('msvcbuild.bat static')
++    os.makedirs(dst_dir + "/lib", exist_ok=True)
++    shutil.copy("lua51.lib", dst_dir + "/lib")
++    os.makedirs(dst_dir + "/include/luajit-2.1", exist_ok=True)
++    for header in ["lauxlib.h", "luaconf.h", "lua.h", "lua.hpp", "luajit.h", "lualib.h"]:
++      shutil.copy(header, dst_dir + "/include/luajit-2.1")
++    os.makedirs(dst_dir + "/bin", exist_ok=True)
++    shutil.copy("luajit.exe", dst_dir + "/bin")
++    
++if os.name == 'nt':
++  win_main()
++else:
++  main()
 +
diff --git a/bazel/foreign_cc/moonjit.patch b/bazel/foreign_cc/moonjit.patch
@@ -48,7 +48,7 @@ new file mode 100755
 index 0000000..9c71271
 --- /dev/null
 +++ b/build.py
-@@ -0,0 +1,39 @@
+@@ -0,0 +1,56 @@
 +#!/usr/bin/env python
 +
 +import argparse
@@ -86,5 +86,22 @@ index 0000000..9c71271
 +
 +    os.system('make V=1 PREFIX="{}" install'.format(args.prefix))
 +
-+main()
++def win_main():
++    src_dir = os.path.dirname(os.path.realpath(__file__))
++    dst_dir = os.getcwd() + "/moonjit"
++    shutil.copytree(src_dir, os.path.basename(src_dir))
++    os.chdir(os.path.basename(src_dir) + "/src")
++    os.system('msvcbuild.bat static')
++    os.makedirs(dst_dir + "/lib", exist_ok=True)
++    shutil.copy("lua51.lib", dst_dir + "/lib")
++    os.makedirs(dst_dir + "/include/moonjit-2.2", exist_ok=True)
++    for header in ["lauxlib.h", "luaconf.h", "lua.h", "lua.hpp", "luajit.h", "lualib.h"]:
++      shutil.copy(header, dst_dir + "/include/moonjit-2.2")
++    os.makedirs(dst_dir + "/bin", exist_ok=True)
++    shutil.copy("luajit.exe", dst_dir + "/bin")
++    
++if os.name == 'nt':
++  win_main()
++else:
++  main()
 +
diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
@@ -5,8 +5,14 @@ load("@envoy_api//bazel:envoy_http_archive.bzl", "envoy_http_archive")
 load(":repository_locations.bzl", "REPOSITORY_LOCATIONS")
 load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language")
 
-# dict of {build recipe name: longform extension name,}
-PPC_SKIP_TARGETS = {"luajit": "envoy.filters.http.lua"}
+PPC_SKIP_TARGETS = ["envoy.filters.http.lua"]
+
+WINDOWS_SKIP_TARGETS = [
+    "envoy.tracers.dynamic_ot",
+    "envoy.tracers.lightstep",
+    "envoy.tracers.datadog",
+    "envoy.tracers.opencensus",
+]
 
 # Make all contents of an external repository accessible under a filegroup.  Used for external HTTP
 # archives, e.g. cares.
@@ -787,6 +793,7 @@ filegroup(
     http_archive(
         name = "kafka_source",
         build_file_content = KAFKASOURCE_BUILD_CONTENT,
+        patches = ["@envoy//bazel/external:kafka_int32.patch"],
         **REPOSITORY_LOCATIONS["kafka_source"]
     )
 

diff --git a/ci/windows_ci_setup.ps1 b/ci/windows_ci_setup.ps1
@@ -2,14 +2,6 @@
 # The list of installed software can be found at:
 # https://github.com/actions/virtual-environments/blob/master/images/win/Windows2019-Readme.md
 
-Add-Type -AssemblyName System.IO.Compression.FileSystem
-function Unzip
-{
-    param([string]$zipfile, [string]$outpath)
-
-    [System.IO.Compression.ZipFile]::ExtractToDirectory($zipfile, $outpath)
-}
-
 function Checksum
 {
     param([string]$filepath, [string]$expected, [string]$algorithm)
@@ -26,10 +18,23 @@ function Checksum
 mkdir "$env:TOOLS_BIN_DIR"
 $wc = New-Object System.Net.WebClient
 $wc.DownloadFile("https://github.com/bazelbuild/bazelisk/releases/download/v1.0/bazelisk-windows-amd64.exe", "$env:TOOLS_BIN_DIR\bazel.exe")
-$wc.DownloadFile("https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-win.zip", "$env:TOOLS_BIN_DIR\ninja-win.zip")
+# See https://sourceforge.net/projects/msys2/files/Base/x86_64/ for msys2 download source
+$wc.DownloadFile("http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20190524.tar.xz", "$env:TEMP\msys2.tar.xz")
 
 # Check the SHA256 file hash of each downloaded file.
 Checksum $env:TOOLS_BIN_DIR\bazel.exe 96395ee9e3fb9f4499fcaffa8a94dd72b0748f495f366bc4be44dbf09d6827fc SHA256
-Checksum $env:TOOLS_BIN_DIR\ninja-win.zip 2d70010633ddaacc3af4ffbd21e22fae90d158674a09e132e06424ba3ab036e9 SHA256
+Checksum $env:TEMP\msys2.tar.xz 168e156fa9f00d90a8445676c023c63be6e82f71487f4e2688ab5cb13b345383 SHA256
 
-Unzip "$env:TOOLS_BIN_DIR\ninja-win.zip" "$env:TOOLS_BIN_DIR"
+# Unpack and install msys2 and required packages
+$tarpath="$env:ProgramFiles\Git\usr\bin\tar.exe"
+$msys2TarPathClean = "/$env:TEMP/msys2.tar.xz".replace(':', '').replace('\', '/')
+$outDirClean = "/$env:TOOLS_BIN_DIR".replace(':', '').replace('\', '/')
+&"$tarpath" -Jxf $msys2TarPathClean -C $outDirClean --strip-components=1
+# Add utils to the path for msys2 setup
+$env:PATH = "$env:TOOLS_BIN_DIR\usr\bin;$env:TOOLS_BIN_DIR\mingw64\bin;$env:PATH"
+bash.exe -c "pacman-key --init 2>&1"
+bash.exe -c "pacman-key --populate msys2 2>&1"
+bash.exe -c "pacman.exe -Syyuu --noconfirm 2>&1"
+bash.exe -c "pacman.exe -Syuu --noconfirm 2>&1"
+bash.exe -c "pacman.exe -S --noconfirm --needed compression diffutils patch 2>&1"
+bash.exe -c "pacman.exe -Scc --noconfirm 2>&1"
diff --git a/ci/windows_ci_steps.sh b/ci/windows_ci_steps.sh
@@ -20,14 +20,22 @@ df -h
 # rules_foreign_cc does not currently use bazel output/temp directories by default, it uses mktemp
 # which respects the value of the TMPDIR environment variable
 drive="$(readlink -f $TMPDIR | cut -d '/' -f2)"
-/c/windows/system32/cmd.exe "/c mklink /d $drive:\\$drive $drive:\\"
+if [ ! -e "/$drive/$drive" ]; then
+  /c/windows/system32/cmd.exe /c "mklink /d $drive:\\$drive $drive:\\"
+fi
 
 BAZEL_STARTUP_OPTIONS="--noworkspace_rc --bazelrc=windows/.bazelrc --output_base=c:/_eb"
 BAZEL_BUILD_OPTIONS="-c opt --config=msvc-cl --show_task_finish --verbose_failures \
   --test_output=all ${BAZEL_BUILD_EXTRA_OPTIONS} ${BAZEL_EXTRA_TEST_OPTIONS}"
 
-bazel ${BAZEL_STARTUP_OPTIONS} build ${BAZEL_BUILD_OPTIONS} //bazel/... --build_tag_filters=-skip_on_windows
+# With all envoy-static and //test/ tree building, no need to test compile externals
+# bazel ${BAZEL_STARTUP_OPTIONS} build ${BAZEL_BUILD_OPTIONS} //bazel/... --build_tag_filters=-skip_on_windows
 
 bazel ${BAZEL_STARTUP_OPTIONS} build ${BAZEL_BUILD_OPTIONS} //source/exe:envoy-static --build_tag_filters=-skip_on_windows
 
-# bazel ${BAZEL_STARTUP_OPTIONS} test ${BAZEL_BUILD_OPTIONS} //test/... --test_tag_filters=-skip_on_windows --build_tests_only --test_summary=terse --test_output=errors
+# Test compilation of known MSVC-compatible test sources
+bazel ${BAZEL_STARTUP_OPTIONS} build ${BAZEL_BUILD_OPTIONS} //test/... --test_tag_filters=-skip_on_windows --build_tests_only
+
+# Test invocations of known-working tests on Windows
+# bazel ${BAZEL_STARTUP_OPTIONS} test ${BAZEL_BUILD_OPTIONS} //test/... --test_tag_filters=-skip_on_windows,-fails_on_windows --build_tests_only --test_summary=terse --test_output=errors
+
diff --git a/source/exe/BUILD b/source/exe/BUILD
@@ -9,12 +9,8 @@ load(
     "envoy_cc_win32_library",
     "envoy_package",
 )
-load(
-    "//source/extensions:all_extensions.bzl",
-    "envoy_all_extensions",
-    "envoy_windows_extensions",
-)
-load("//bazel:repositories.bzl", "PPC_SKIP_TARGETS")
+load("//source/extensions:all_extensions.bzl", "envoy_all_extensions")
+load("//bazel:repositories.bzl", "PPC_SKIP_TARGETS", "WINDOWS_SKIP_TARGETS")
 
 envoy_package()
 
@@ -41,7 +37,7 @@ envoy_cc_library(
         "//source/server:server_lib",
         "//source/server:listener_hooks_lib",
     ] + select({
-        "//bazel:windows_x86_64": envoy_windows_extensions(),
+        "//bazel:windows_x86_64": envoy_all_extensions(WINDOWS_SKIP_TARGETS),
         "//bazel:linux_ppc": envoy_all_extensions(PPC_SKIP_TARGETS),
         "//conditions:default": envoy_all_extensions(),
     }),

diff --git a/source/exe/win32/platform_impl.cc b/source/exe/win32/platform_impl.cc
@@ -11,8 +11,7 @@ PlatformImpl::PlatformImpl()
       file_system_(std::make_unique<Filesystem::InstanceImplWin32>()) {
   const WORD wVersionRequested = MAKEWORD(2, 2);
   WSADATA wsaData;
-  const int rc = ::WSAStartup(wVersionRequested, &wsaData);
-  RELEASE_ASSERT(rc == 0, "WSAStartup failed with error");
+  RELEASE_ASSERT(WSAStartup(wVersionRequested, &wsaData) == 0, "WSAStartup failed with error");
 }
 
 PlatformImpl::~PlatformImpl() { ::WSACleanup(); }

diff --git a/source/extensions/all_extensions.bzl b/source/extensions/all_extensions.bzl
@@ -1,5 +1,5 @@
 load("@bazel_skylib//lib:dicts.bzl", "dicts")
-load("@envoy_build_config//:extensions_build_config.bzl", "EXTENSIONS", "WINDOWS_EXTENSIONS")
+load("@envoy_build_config//:extensions_build_config.bzl", "EXTENSIONS")
 
 # These extensions are registered using the extension system but are required for the core Envoy build.
 # The map may be overridden by extensions specified in envoy_build_config.
@@ -9,14 +9,8 @@ _required_extensions = {
 }
 
 # Return all extensions to be compiled into Envoy.
-def envoy_all_extensions(blacklist = dict()):
+def envoy_all_extensions(blacklist = []):
     all_extensions = dicts.add(_required_extensions, EXTENSIONS)
 
     # These extensions can be removed on a site specific basis.
-    return [v for k, v in all_extensions.items() if not k in blacklist.values()]
-
-def envoy_windows_extensions():
-    all_extensions = dicts.add(_required_extensions, WINDOWS_EXTENSIONS)
-
-    # These extensions can be removed on a site specific basis.
-    return all_extensions.values()
+    return [v for k, v in all_extensions.items() if not k in blacklist]
diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc
@@ -1,7 +1,5 @@
 #include "redis_cluster.h"
 
-#include <err.h>
-
 #include "envoy/config/cluster/redis/redis_cluster.pb.h"
 #include "envoy/config/cluster/redis/redis_cluster.pb.validate.h"
 #include "envoy/config/cluster/v3/cluster.pb.h"