From b3824d0388101b32eba9930ebed38cc17805fa1c Mon Sep 17 00:00:00 2001 From: Yevgeni Tsodikov Date: Tue, 9 Dec 2025 14:47:25 +0200 Subject: [PATCH 1/4] Add generic protoc plugin support (v1.1.0) - Add :plugins configuration option for generic protoc plugins - Auto-download plugins from configurable URL templates with platform detection - Support plugin-specific options (e.g., lang=java for protoc-gen-validate) - Add :additional-flags for plugins requiring extra protoc flags (e.g., protoc-gen-doc) - Unified plugin installation directory for all plugins including gRPC - Plugin downloads try multiple platform naming conventions automatically - Handle .tar.gz, .gz, and plain executable formats - Backward compatible with existing :compile-grpc? and :grpc-version config - Add comprehensive test coverage for plugin functionality --- CHANGELOG.md | 24 ++ README.md | 192 ++++++++++++--- project.clj | 2 +- src/leiningen/protodeps.clj | 393 +++++++++++++++++++----------- test/leiningen/protodeps_test.clj | 115 ++++++++- 5 files changed, 541 insertions(+), 185 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14e5502..d29567d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.1.0] - 2025-12-09 + +### Added +* Generic protoc plugin support via `:plugins` configuration option +* Auto-download plugins from configurable URL templates +* Plugin-specific options support (e.g., `lang=java` for protoc-gen-validate) +* Multiple plugins can run simultaneously +* Test coverage for plugin functionality +* Plugin `:additional-flags` configuration option for plugins requiring extra protoc flags (e.g., `protoc-gen-doc`'s `--doc_opt`) + +### Changed +* `protoc-opts` function now accepts a vector of plugins +* Legacy gRPC configuration converted to plugin format internally +* Unified plugin installation directory to `~/.lein-protodeps/plugins-installations///` for all plugins including gRPC +* Plugin downloads try multiple platform naming conventions automatically (`osx`/`darwin`, `aarch_64`/`arm64`) until one succeeds + +### Removed +* Removed `get-grpc-plugin!` and `download-grpc-plugin!` functions + +### Fixed +* Plugin binaries always have execute permissions set, even when previously downloaded +* Plugin downloads handle `.tar.gz`, `.gz`, and plain executable formats +* Plugin downloads try all platform name variants before failing + ## [1.0.6] - 2025-08-07 ### Fixed diff --git a/README.md b/README.md index d1e0d8e..7cc8f40 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ are not already installed. - [Usage](#usage) - [Cross-repository compilation](#cross-repository-compilation) - [Git HTTP authentication](#git-http-authentication) +- [Protoc Plugins](#protoc-plugins) - [protoc and gRPC binaries retrieval](#protoc-and-grpc-binaries-retrieval) - [Configuration Reference](#configuration-reference) - [Plugin options](#plugin-configuration-options) @@ -30,7 +31,7 @@ are not already installed. ## Usage -Put `[com.appsflyer/lein-protodeps "1.0.6"]` into the `:plugins` vector of your project.clj. +Put `[com.appsflyer/lein-protodeps "1.1.0"]` into the `:plugins` vector of your project.clj. Once installed, run `lein protodeps generate` to run the plugin. @@ -50,6 +51,12 @@ An example configuration: :proto-version ~proto-version :grpc-version ~grpc-version :compile-grpc? true ;; whether to activate the gRPC plugin during the stub generation process + ;; Optional: protoc plugins for validation, documentation, etc. + :plugins [{:name "protoc-gen-validate" + :version "1.3.0" + :url-template "https://github.com/bufbuild/protoc-gen-validate/releases/download/v${:version}/protoc-gen-validate_${:version}_${:os-name}_${:os-arch}.tar.gz" + :output-directive "validate_out" + :options {:lang "java"}}] ;; Repositories configuration. Each entry in this map is an entry mapping a logical repository name ;; to its configuration. :repos {:af-schemas {:repo-type :git ;; a git repo @@ -86,53 +93,168 @@ To use HTTP authentication using username and password, provide them in the clon It is recommended to use environment variables rather than hardcoding their values in plaintext. Environment variables are accessible via the `${:env/}` interpolation syntax, which allows us to write the former as: `"https://${:env/GIT_USERNAME}:${:env/GIT_PASSWORD}@github.com/whatever/cool_repo.git"`. +## Protoc Plugins + +`lein-protodeps` supports protoc plugins (e.g., `protoc-gen-validate`, `protoc-gen-doc`) in addition to gRPC. + +### Using Protoc Plugins + +Add a `:plugins` vector to your configuration: + +```clj +:lein-protodeps {:output-path "src/java/generated" + :proto-version "3.12.4" + :grpc-version "1.68.1" + :compile-grpc? true ; gRPC via legacy config + :plugins [{:name "protoc-gen-validate" + :version "1.3.0" + :url-template "https://github.com/bufbuild/protoc-gen-validate/releases/download/v${:version}/protoc-gen-validate_${:version}_${:os-name}_${:os-arch}.tar.gz" + :output-directive "validate_out" + :options {:lang "java"}}] + :repos {...}} +``` + +### Plugin Configuration Keys + +- `:name` (required) - The plugin executable name (e.g., `"protoc-gen-validate"`) +- `:version` (required) - The plugin version to download +- `:url-template` (required) - URL template for downloading the plugin binary. Interpolation variables: + - `${:version}` - Plugin version (`${:semver}` also works for backward compatibility) + - `${:os-name}` - Host OS name (tries `osx`/`darwin` for macOS, `linux` for Linux) + - `${:os-arch}` - Host architecture (tries `x86_64`/`amd64` for x64, `aarch_64`/`arm64` for ARM64) +- `:output-directive` (required) - The protoc output flag name without `--` (e.g., `"validate_out"` becomes `--validate_out`) +- `:options` (optional) - Map of plugin-specific options formatted as `key=value` pairs in the output directive +- `:additional-flags` (optional) - Map of additional protoc flags (e.g., `{:doc_opt "markdown,docs.md"}` becomes `--doc_opt=markdown,docs.md`) + +The library tries multiple naming conventions automatically. On macOS ARM64, it generates and tries URLs with `osx`/`aarch_64`, then `darwin`/`arm64` until one succeeds. + +### Plugin Installation + +Plugins are automatically downloaded and installed to `~/.lein-protodeps/plugins-installations///` when first used. + +**Supported formats:** +- `.tar.gz` archives (extracts the plugin binary from the tarball) +- `.gz` compressed files (decompresses to binary) +- Plain executables (downloads directly) + +Execute permissions are set automatically on all downloaded binaries. + +### Example: Using protoc-gen-validate + +```clj +:plugins [{:name "protoc-gen-validate" + :version "1.3.0" + :url-template "https://github.com/bufbuild/protoc-gen-validate/releases/download/v${:version}/protoc-gen-validate_${:version}_${:os-name}_${:os-arch}.tar.gz" + :output-directive "validate_out" + :options {:lang "java"}}] +``` + +On macOS ARM64, the library tries these URLs in order: +1. `protoc-gen-validate_1.3.0_osx_aarch_64.tar.gz` (protoc style) - not found +2. `protoc-gen-validate_1.3.0_osx_arm64.tar.gz` - not found +3. `protoc-gen-validate_1.3.0_darwin_aarch_64.tar.gz` - not found +4. `protoc-gen-validate_1.3.0_darwin_arm64.tar.gz` (Go style) - success! + +Generated protoc command: +```bash +protoc --proto_path=... \ + --java_out=output/ \ + --plugin=protoc-gen-validate=~/.lein-protodeps/plugins-installations/protoc-gen-validate/1.3.0/protoc-gen-validate \ + --validate_out=lang=java:output/ \ + file.proto +``` + +### Using Multiple Plugins + +You can configure multiple plugins together. For example, using gRPC, validation, and documentation: + +```clj +:lein-protodeps {:output-path "src/java/generated" + :proto-version "3.25.5" + :grpc-version "1.68.1" + :compile-grpc? true ; Adds gRPC plugin + :plugins [{:name "protoc-gen-validate" + :version "1.3.0" + :url-template "https://github.com/bufbuild/protoc-gen-validate/releases/download/v${:version}/protoc-gen-validate_${:version}_${:os-name}_${:os-arch}.tar.gz" + :output-directive "validate_out" + :options {:lang "java"}} + {:name "protoc-gen-doc" + :version "1.5.1" + :url-template "https://github.com/pseudomuto/protoc-gen-doc/releases/download/v${:version}/protoc-gen-doc_${:version}_${:os-name}_${:os-arch}.tar.gz" + :output-directive "doc_out" + :additional-flags {:doc_opt "markdown,docs.md"}}] + :repos {...}} +``` + +The [protoc-gen-doc](https://github.com/pseudomuto/protoc-gen-doc) plugin uses `:additional-flags` for format configuration since it requires the separate `--doc_opt` flag. + +### Backward Compatibility + +The `:compile-grpc?` and `:grpc-version` options are syntactic sugar for adding gRPC to the `:plugins` vector. These two configurations are equivalent: + +```clj +;; Using legacy config (syntactic sugar) +:compile-grpc? true +:grpc-version "1.68.1" + +;; Equivalent plugin config +:plugins [{:name "protoc-gen-grpc-java" + :version "1.68.1" + :url-template "https://repo1.maven.org/maven2/io/grpc/protoc-gen-grpc-java/${:version}/protoc-gen-grpc-java-${:version}-${:os-name}_${:os-arch}.exe" + :output-directive "grpc-java_out"}] +``` + +You can use both forms together. When `:compile-grpc?` is true, gRPC is automatically added to the plugins list. + ## protoc and gRPC binaries retrieval -The plugin will download the protoc and gRPC plugin binaries according to the versions set in `:proto-version` and `:grpc-version`, respectively, and install them under -`~/.lein-protodeps/`. +The plugin downloads protoc and gRPC binaries based on `:proto-version` and `:grpc-version`, installing them to `~/.lein-protodeps/`. + +Default download locations: +- protoc: `https://github.com/protocolbuffers/protobuf/releases/download/` +- gRPC: `https://repo1.maven.org/maven2/io/grpc/protoc-gen-grpc-java/` -By default, the plugin will use `https://github.com/protocolbuffers/protobuf/releases/download/` for protoc and `https://repo1.maven.org/maven2/io/grpc/protoc-gen-grpc-java/` for gRPC. -However, it is possible to override these to other endpoints by setting `:protoc-zip-url-template` and `:grpc-exe-url-template` options in the plugin configuration. +Override these by setting `:protoc-zip-url-template` or `:grpc-exe-url-template`. -The values of these options are URL templates that will be interpolated at runtime with the following variables to produce download URLs: +URL templates support these interpolation variables: -* `:os-name` host OS (i.e, `linux`, `osx`) -* `:os-arch` host architecture (i.e, `x86_64`, `aarch_64`). Note: `aarch64` systems are automatically mapped to `aarch_64` to match protoc binary naming conventions -* `:semver` version string as defined in `:protoc-version` or `:grpc-version` -* `:major` major part of `:semver` -* `:minor` minor part of `:semver` -* `:patch` patch part of `:semver` +* `${:version}` or `${:semver}` - version string from `:protoc-version` or `:grpc-version` +* `${:major}`, `${:minor}`, `${:patch}` - version components +* `${:os-name}` - host OS (`linux`, `osx`) +* `${:os-arch}` - host architecture (`x86_64`, `aarch_64`) -For example, to override the gRPC URL you may set `:grpc-exe-url-template` to `https://some-other-place.com/artifacts/grpc/${:semver}/protoc-gen-grpc-java-${:semver}-${:os-name}-${:os-arch}`. Note that -currently this feature does not support any method of authentication, in case your endpoints require it. +Example override: +```clj +:grpc-exe-url-template "https://my-mirror.com/grpc/${:version}/protoc-gen-grpc-java-${:version}-${:os-name}_${:os-arch}.exe" +``` + +Note: Authentication is not supported for custom URLs. ## Configuration Reference #### Plugin Configuration Options -| Key | Type | Req? | Notes | -|----------------------------|---------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------| -| `:output-path` | string | required | Path to which to compile stubs. This usually needs to be under `:java-source-paths` | -| `:proto-version` | string | required | Version of protoc to use | -| `:grpc-version` | string | optional | Version of gRPC plugin to use. Required if `:compile-grpc?` is `true` | -| `:compile-grpc?` | boolean | optional | Whether to compile gRPC stubs. Defaults to `false` | -| `:protoc-zip-url-template` | string | optional | URL template from which to retrieve protoc's zip release (if needed). See also [protoc and gRPC binaries retrieval](#protoc-and-grpc-binaries-retrieval) | -| `:grpc-exe-url-template` | string | optional | URL template from which to retrieve gRPC's executable release (if needed). See also [protoc and gRPC binaries retrieval](#protoc-and-grpc-binaries-retrieval) | +| Key | Type | Req? | Notes | +|----------------------------|----------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `:output-path` | string | required | Path to which to compile stubs. This usually needs to be under `:java-source-paths` | +| `:proto-version` | string | required | Version of protoc to use | +| `:grpc-version` | string | optional | Version of gRPC plugin to use. Required if `:compile-grpc?` is `true` | +| `:compile-grpc?` | boolean | optional | Whether to compile gRPC stubs. Defaults to `false` | +| `:plugins` | vector of maps | optional | Generic protoc plugins configuration. See [Protoc Plugins](#protoc-plugins) section for details | +| `:protoc-zip-url-template` | string | optional | URL template from which to retrieve protoc's zip release (if needed). See also [protoc and gRPC binaries retrieval](#protoc-and-grpc-binaries-retrieval) | +| `:grpc-exe-url-template` | string | optional | URL template from which to retrieve gRPC's executable release (if needed). See also [protoc and gRPC binaries retrieval](#protoc-and-grpc-binaries-retrieval) | #### Repo Options -| Key | Type | Req? | Notes | -|----------------------------------|----------------------|---------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------| -| `<:repo-name>.:repo-type` | `:git` `:filesystem` | required | | -| `<:repo-name>.:config.:clone-url` | string | required (for git repos) | Either SSH or HTTP endpoints are supported, see also [Git HTTP authentication](#git-http-authentication) | -| `<:repo-name>.:config.:rev` | string | optional (for git repos) | commit hash/tag name/branch name. Not specifying a rev will default to cloning the main branch (it is generally encouraged to use a fixed version) | -| `<:repo-name>.:config.:path` | string | required (for filesystem repos) | path to directory containing files (absolute or relative to project directory) | +| Key | Type | Req? | Notes | +|-----------------------------------|----------------------|---------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------| +| `<:repo-name>.:repo-type` | `:git` `:filesystem` | required | | +| `<:repo-name>.:config.:clone-url` | string | required (for git repos) | Either SSH or HTTP endpoints are supported, see also [Git HTTP authentication](#git-http-authentication) | +| `<:repo-name>.:config.:rev` | string | optional (for git repos) | commit hash/tag name/branch name. Not specifying a rev will default to cloning the main branch (it is generally encouraged to use a fixed version) | +| `<:repo-name>.:config.:path` | string | required (for filesystem repos) | path to directory containing files (absolute or relative to project directory) | #### Proto options -| Key | Type | Req? | Notes | -|------------------------------|-------------------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `<:repo-name>.:proto-paths` | vector of strings | required | Relative proto paths to the directory root (where `protoc` will search for imports, see `protoc --help` for more information) | -| `<:repo-name>.:dependencies` | vector of symbols | optional | List of paths which contain files to compile to stubs. Each of these paths needs to be prefixed with one of the proto paths defined under `:proto-paths`, see [example](#usage) above. If unspecified or empty, nothing in this repo will be compiled, but it may still be used for finding imports required by other repos. See also [cross-repository compilation](#cross-repository-compilation) | - - - +| Key | Type | Req? | Notes | +|------------------------------|-------------------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `<:repo-name>.:proto-paths` | vector of strings | required | Relative proto paths to the directory root (where `protoc` will search for imports, see `protoc --help` for more information) | +| `<:repo-name>.:dependencies` | vector of symbols | optional | List of paths which contain files to compile to stubs. Each of these paths needs to be prefixed with one of the proto paths defined under `:proto-paths`, see [example](#usage) above. If unspecified or empty, nothing in this repo will be compiled, but it may still be used for finding imports required by other repos. See also [cross-repository compilation](#cross-repository-compilation) | diff --git a/project.clj b/project.clj index e412ad5..fab0f0b 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.appsflyer/lein-protodeps "1.0.7" +(defproject com.appsflyer/lein-protodeps "1.1.0-SNAPSHOT" :description "Leiningen plugin for consuming and compiling protobuf schemas" :url "https://github.com/AppsFlyer/lein-protodeps" :license {:name "Apache License, Version 2.0" diff --git a/src/leiningen/protodeps.clj b/src/leiningen/protodeps.clj index ac61823..7517ff6 100644 --- a/src/leiningen/protodeps.clj +++ b/src/leiningen/protodeps.clj @@ -1,14 +1,16 @@ (ns leiningen.protodeps (:require [clojure.java.io :as io] - [clojure.string :as strings] [clojure.java.shell :as sh] [clojure.set :as sets] - [leiningen.core.main :as lein] - [clojure.tools.cli :as cli]) - (:import [java.io File] - [java.nio.file Files] - [java.nio.file.attribute FileAttribute] - [java.nio.file Path])) + [clojure.string :as strings] + [clojure.tools.cli :as cli] + [leiningen.core.main :as lein]) + (:import (java.io File FileNotFoundException) + (java.nio.file Files) + (java.nio.file Path) + (java.nio.file.attribute FileAttribute PosixFilePermission) + (java.util HashSet) + (java.util.zip GZIPInputStream ZipEntry ZipInputStream))) (def ^:dynamic *verbose?* false) @@ -100,7 +102,7 @@ rev (:rev git-config)] (println "cloning" repo-name "at rev" rev "...") (when-not rev - (throw (ex-info (str ":rev is not set for " repo-name ", set a git tag/branch name/commit hash") {}))) + (throw (ex-info (str ":rev is not set for " repo-name ", set a git tag/branch name/commit hash") {}))) (git-clone! repo-url path rev) path)) @@ -113,8 +115,8 @@ (defmethod resolve-repo :filesystem [_ repo-config] (some-> repo-config :config :path io/file .getAbsolutePath)) -(defn write-zip-entry! [^java.util.zip.ZipInputStream zinp - ^java.util.zip.ZipEntry entry +(defn write-zip-entry! [^ZipInputStream zinp + ^ZipEntry entry base-path] (let [file-name (append-dir base-path (.getName entry)) ^File file (io/file file-name) @@ -130,15 +132,19 @@ (.write outp buf 0 bytes-read) (recur))))))))) -(defn unzip! [^java.util.zip.ZipInputStream zinp dst] +(defn unzip! [^ZipInputStream zinp dst] (loop [] - (when-let [^java.util.zip.ZipEntry entry (.getNextEntry zinp)] + (when-let [entry (.getNextEntry zinp)] (write-zip-entry! zinp entry dst) (.closeEntry zinp) (recur)))) -(def os-name->os {"Linux" "linux" "Mac OS X" "osx"}) -(def os-arch->arch {"amd64" "x86_64" "x86_64" "x86_64" "aarch64" "aarch_64"}) +(def os-name->os {"Linux" ["linux"] + "Mac OS X" ["osx" "darwin"]}) + +(def os-arch->arch {"amd64" ["x86_64" "amd64"] + "x86_64" ["x86_64" "amd64"] + "aarch64" ["aarch_64" "arm64"]}) (defn get-prop [env prop-name] @@ -153,45 +159,83 @@ (defn- get-platform [env] - (let [raw-os-name (get env "os.name") - raw-os-arch (get env "os.arch") - os-name (get os-name->os raw-os-name) - os-arch (get os-arch->arch raw-os-arch) - platform {:os-name os-name :os-arch os-arch}] - (when (or (nil? os-arch) (nil? os-name)) + (let [raw-os-name (get env "os.name") + raw-os-arch (get env "os.arch") + os-variants (get os-name->os raw-os-name) + arch-variants (get os-arch->arch raw-os-arch)] + (when (or (nil? os-variants) (nil? arch-variants)) (throw (ex-info "\nPlatform is not currently supported.\n Please open an issue at https://github.com/AppsFlyer/lein-protodeps/issues and include this full error message to add support for your platform." - {:os.name raw-os-name :os.arch raw-os-arch}))) - (get platform-alternatives platform platform))) + {:os.name raw-os-name :os.arch raw-os-arch}))) + ; Use first variant as default (for backward compatability and protoc URLs) + ; Store all variants for plugin downloads that may need alternatives + (let [platform {:os-name (first os-variants) + :os-arch (first arch-variants) + :os-name-variants os-variants + :os-arch-variants arch-variants + :semver nil}] ; Will be set later for proto/grpc versions + (get platform-alternatives platform platform)))) (defn- get-protoc-release [{:keys [semver os-name os-arch]}] (strings/join "-" ["protoc" semver os-name os-arch])) -(def ^:private grpc-plugin-executable-name "protoc-gen-grpc-java") - - (defn set-protoc-permissions! [protoc-path] - (let [permissions (java.util.HashSet.)] - (.add permissions java.nio.file.attribute.PosixFilePermission/OWNER_EXECUTE) - (.add permissions java.nio.file.attribute.PosixFilePermission/OWNER_READ) - (.add permissions java.nio.file.attribute.PosixFilePermission/OWNER_WRITE) - (java.nio.file.Files/setPosixFilePermissions (.toPath (io/file protoc-path)) - permissions))) - + (let [permissions (HashSet.)] + (.add permissions PosixFilePermission/OWNER_EXECUTE) + (.add permissions PosixFilePermission/OWNER_READ) + (.add permissions PosixFilePermission/OWNER_WRITE) + (Files/setPosixFilePermissions (.toPath (io/file protoc-path)) + permissions))) (defn download-protoc! [url dst] - (println "protodeps: Downloading protoc from" url "...") - (with-open [inp (java.util.zip.ZipInputStream. (io/input-stream url))] + (println "protodeps: Downloading protoc from" url "to" dst "...") + (with-open [inp (ZipInputStream. (io/input-stream url))] (unzip! inp dst))) -(defn- download-grpc-plugin! [url grpc-plugin-file] - (println "protodeps: Downloading grpc java plugin from" url "...") - (with-open [input-stream (io/input-stream url) - output-stream (io/output-stream (io/file grpc-plugin-file))] - (io/copy input-stream output-stream))) - +(defn- try-download-plugin! + "Attempt to download a plugin from a single URL. Returns true on success, false on failure." + [plugin-name url plugin-file] + (try + (println "protodeps: Trying to download" plugin-name "from" url "...") + (if (strings/ends-with? url ".tar.gz") + (let [plugin-dir (.getParent (io/file plugin-file)) + process (-> (ProcessBuilder. ["tar" "-xzf" "-" "-C" plugin-dir plugin-name]) + (.redirectErrorStream true) + (.start))] + (with-open [input (io/input-stream url) + output (.getOutputStream process)] + (io/copy input output)) + (let [exit (.waitFor process)] + (when-not (zero? exit) + (let [err (slurp (.getInputStream process))] + (println "protodeps: Failed to extract tar.gz:" err) + (throw (ex-info "Failed to extract tar.gz" {:exit exit :error err})))))) + (with-open [raw-input (io/input-stream url) + input (if (strings/ends-with? url ".gz") + (GZIPInputStream. raw-input) + raw-input) + output (io/output-stream (io/file plugin-file))] + (io/copy input output))) + (println "protodeps: Successfully downloaded" plugin-name "to" plugin-file) + true + (catch FileNotFoundException _ + (println "protodeps: URL not found:" url) + false) + (catch Exception e + (println "protodeps: Failed to download/extract from" url ":" (.getMessage e)) + false))) + +(defn- download-plugin! + "Try downloading plugin from multiple URL variants until one succeeds. + Throws an exception if all URLs fail." + [plugin-name urls plugin-file] + (loop [[url & remaining] urls] + (if url + (when (not (try-download-plugin! plugin-name url plugin-file)) + (recur remaining)) + (throw (ex-info "Failed to download plugin from any URL" {:plugin plugin-name :urls urls}))))) (defn run-protoc-and-report! [protoc-path opts] (let [{:keys [out err]} (run-sh! protoc-path opts)] @@ -207,7 +251,7 @@ and include this full error message to add support for your platform." (def new-protoc-release-tpl "https://github.com/protocolbuffers/protobuf/releases/download/v${:minor}.${:patch}/protoc-${:minor}.${:patch}-${:os-name}-${:os-arch}.zip") -(def grpc-release-tpl "https://repo1.maven.org/maven2/io/grpc/protoc-gen-grpc-java/${:semver}/protoc-gen-grpc-java-${:semver}-${:os-name}-${:os-arch}.exe") +(def grpc-release-tpl "https://repo1.maven.org/maven2/io/grpc/protoc-gen-grpc-java/${:version}/protoc-gen-grpc-java-${:version}-${:os-name}_${:os-arch}.exe") (defn- protoc-release-template [{:keys [protoc-zip-url-template]} @@ -222,21 +266,21 @@ and include this full error message to add support for your platform." (def ^:private protoc-install-dir "protoc-installations") -(def ^:private grpc-install-dir "grpc-installations") +(def ^:private plugins-install-dir "plugins-installations") (defn init-rc-dir! [] (let [home (append-dir (get-prop (System/getProperties) "user.home") ".lein-protodeps")] (mkdir! home) (mkdir! (append-dir home protoc-install-dir)) - (mkdir! (append-dir home grpc-install-dir)) + (mkdir! (append-dir home plugins-install-dir)) home)) (defn discover-files [git-repo-path dep-path] (filterv - (fn [^File file] - (and (not (.isDirectory file)) - (strings/ends-with? (.getName file) ".proto"))) - (file-seq (io/file (append-dir git-repo-path dep-path))))) + (fn [^File file] + (and (not (.isDirectory file)) + (strings/ends-with? (.getName file) ".proto"))) + (file-seq (io/file (append-dir git-repo-path dep-path))))) (defn long-opt [k v] (str "--" k "=" v)) @@ -250,49 +294,49 @@ and include this full error message to add support for your platform." (map io/file (re-seq #"[^\s]*\.proto" (:out - (run-sh! - protoc-path - (with-proto-paths - [(long-opt "dependency_out" "/dev/stdout") - "-o/dev/null" - (.getAbsolutePath proto-file)] - proto-paths)))))) + (run-sh! + protoc-path + (with-proto-paths + [(long-opt "dependency_out" "/dev/stdout") + "-o/dev/null" + (.getAbsolutePath proto-file)] + proto-paths)))))) (defn parallelize [{:keys [level min-chunk-size]} c combine-f f] (if (or (= 1 level) (< (count c) min-chunk-size)) (f c) - (let [chunks (partition-all (int (/ (count c) level)) c) + (let [chunks (partition-all (int (/ (count c) level)) c) ;; parallelism is capped by number of cores results (pmap f chunks)] (transduce (map identity) combine-f results)))) (defn expand-dependencies [parallelism protoc-path proto-paths proto-files] (parallelize - parallelism - proto-files - sets/union - (fn [proto-files] - (loop [seen-files (set proto-files) - [f & r] proto-files] - (if-not f - seen-files - (let [deps (get-file-dependencies protoc-path proto-paths f) - deps (filterv - (fn [^File afile] - (not (some #(Files/isSameFile (.toPath ^File %) (.toPath afile)) seen-files))) - deps)] - (recur - (conj seen-files f) - ;; For very large repos, we might end up concatenating an empty `deps` seq - ;; many times over since most of the depenendencies will already be seen in prev iterations. - ;; This could lead to the build up of a huge lazy-seq, since `concat` will still - ;; cons to the seq. To circumvent this we concat only when non-empty. - (if-not (seq deps) - r - (concat - r - deps))))))))) + parallelism + proto-files + sets/union + (fn [proto-files] + (loop [seen-files (set proto-files) + [f & r] proto-files] + (if-not f + seen-files + (let [deps (get-file-dependencies protoc-path proto-paths f) + deps (filterv + (fn [^File afile] + (not (some #(Files/isSameFile (.toPath ^File %) (.toPath afile)) seen-files))) + deps)] + (recur + (conj seen-files f) + ;; For very large repos, we might end up concatenating an empty `deps` seq + ;; many times over since most of the depenendencies will already be seen in prev iterations. + ;; This could lead to the build up of a huge lazy-seq, since `concat` will still + ;; cons to the seq. To circumvent this we concat only when non-empty. + (if-not (seq deps) + r + (concat + r + deps))))))))) (defn strip-suffix [suffix s] (if (strings/ends-with? s suffix) @@ -314,13 +358,36 @@ and include this full error message to add support for your platform." (doseq [file (reverse (file-seq (.toFile path)))] (.delete ^File file))) - -(defn protoc-opts [proto-paths output-path compile-grpc? grpc-plugin ^File proto-file] - (let [protoc-opts (with-proto-paths [(long-opt "java_out" output-path)] proto-paths)] - (cond-> protoc-opts - compile-grpc? (conj (long-opt "grpc-java_out" output-path)) - compile-grpc? (conj (long-opt "plugin" grpc-plugin)) - true (conj (.getAbsolutePath proto-file))))) +(defn- format-plugin-options + "Convert a map of plugin options to a semicolon-separated key=value string" + [options] + (when (seq options) + (strings/join "," (map (fn [[k v]] (str (name k) "=" v)) options)))) + +(defn- add-plugin-opts + "Add protoc options for a single plugin" + [protoc-opts output-path {:keys [plugin-path output-directive options additional-flags]}] + (let [formatted-opts (format-plugin-options options) + output-value (if formatted-opts + (str formatted-opts ":" output-path) + output-path) + additional (when additional-flags + (mapv (fn [[k v]] (long-opt (name k) v)) additional-flags))] + (-> protoc-opts + (conj (long-opt "plugin" plugin-path)) + (conj (long-opt output-directive output-value)) + (into (or additional []))))) + +(defn protoc-opts + "Build protoc command options with support for multiple plugins. + plugins should be a vector of maps with :plugin-path, :output-directive, and optional :options" + [proto-paths output-path plugins ^File proto-file] + (let [base-opts (with-proto-paths [(long-opt "java_out" output-path)] proto-paths)] + (-> (reduce (fn [opts plugin] + (add-plugin-opts opts output-path plugin)) + base-opts + plugins) + (conj (.getAbsolutePath proto-file))))) (def cli-spec [["-h" "--help"] @@ -334,31 +401,68 @@ and include this full error message to add support for your platform." (defn- get-protoc! [home-dir config proto-version] (let [protoc-installs (append-dir home-dir protoc-install-dir) protoc-release (get-protoc-release proto-version) - protoc (append-dir protoc-installs protoc-release "bin" "protoc")] + protoc (append-dir protoc-installs protoc-release "bin" "protoc")] (when-not (.exists ^File (io/file protoc)) (let [protoc-zip-url (interpolate proto-version (protoc-release-template config proto-version))] (download-protoc! protoc-zip-url (append-dir protoc-installs protoc-release))) (set-protoc-permissions! protoc)) protoc)) +(defn- generate-plugin-urls + "Generate all URL variants by combining os-name and os-arch variants" + [url-template platform plugin-version] + (let [os-variants (:os-name-variants platform) + arch-variants (:os-arch-variants platform)] + (for [os-name os-variants + os-arch arch-variants] + (interpolate (merge platform {:version plugin-version + :semver plugin-version ; For backward compat with grpc-release-tpl + :os-name os-name + :os-arch os-arch}) + url-template)))) + +(defn- get-plugin! + "Download and install a protoc plugin if not already present. + plugin-config should have :name, :version, :url-template" + [home-dir platform plugin-config] + (let [plugin-name (:name plugin-config) + plugin-version (:version plugin-config) + install-base-dir (append-dir home-dir plugins-install-dir plugin-name) + plugin-dir (append-dir install-base-dir plugin-version) + plugin-path (append-dir plugin-dir plugin-name)] + (when-not (.exists ^File (io/file plugin-path)) + (mkdir! plugin-dir) + (let [plugin-urls (generate-plugin-urls (:url-template plugin-config) platform plugin-version)] + (download-plugin! plugin-name plugin-urls plugin-path))) + (set-protoc-permissions! plugin-path) + plugin-path)) + +(defn- merge-legacy-grpc-config + "Convert legacy :compile-grpc? and :grpc-version config to new plugin format. + Merges with any plugins specified in :plugins config." + [config grpc-version] + (let [configured-plugins (or (:plugins config) []) + grpc-plugin-config (when (and (:compile-grpc? config) grpc-version) + {:name "protoc-gen-grpc-java" + :version (:semver grpc-version) ; Extract semver string from parsed version + :url-template (or (:grpc-exe-url-template config) + grpc-release-tpl) + :output-directive "grpc-java_out"})] + (cond + grpc-plugin-config + (cons grpc-plugin-config configured-plugins) + + (and (:compile-grpc? config) (not grpc-version)) + (do + (print-warning ":compile-grpc? is true but :grpc-version is not set. gRPC stubs will not be generated.") + configured-plugins) -(defn- get-grpc-plugin! [home-dir config grpc-version] - (let [grpc-semver (:semver grpc-version) - grpc-installs (append-dir home-dir grpc-install-dir) - grpc-plugin-dir (append-dir grpc-installs grpc-semver) - grpc-plugin (append-dir grpc-plugin-dir grpc-plugin-executable-name)] - (when (:compile-grpc? config) - (when (not (.exists ^File (io/file grpc-plugin))) - (mkdir! grpc-plugin-dir) - (let [grpc-exe-url (interpolate grpc-version (or (:grpc-exe-url-template config) - grpc-release-tpl))] - (download-grpc-plugin! grpc-exe-url grpc-plugin)) - (set-protoc-permissions! grpc-plugin)) - grpc-plugin))) + :else + configured-plugins))) (defn generate-files! [opts config] (let [home-dir (init-rc-dir!) - parallelism {:level (:parallelism opts) + parallelism {:level (:parallelism opts) :min-chunk-size 128} repos-config (:repos config) output-path (:output-path config) @@ -368,9 +472,19 @@ and include this full error message to add support for your platform." env (System/getProperties) platform (get-platform env) proto-version (merge platform (parse-semver (:proto-version config))) - grpc-version (merge platform (parse-semver (:grpc-version config))) + grpc-version (when (:grpc-version config) + (merge platform (parse-semver (:grpc-version config)))) protoc (get-protoc! home-dir config proto-version) - grpc-plugin (get-grpc-plugin! home-dir config grpc-version) + ; Merge legacy gRPC config with new plugins config + plugin-configs (merge-legacy-grpc-config config grpc-version) + ; Download and prepare all plugins + plugins (mapv (fn [plugin-config] + (let [plugin-path (get-plugin! home-dir platform plugin-config)] + {:plugin-path plugin-path + :output-directive (:output-directive plugin-config) + :options (:options plugin-config) + :additional-flags (:additional-flags plugin-config)})) + plugin-configs) repo-id->repo-path (into {} (map (fn [[k v]] @@ -384,43 +498,42 @@ and include this full error message to add support for your platform." (try (mkdir! output-path) (verbose-prn "config: %s" config) - (verbose-prn "paths: %s" {:protoc protoc - :grpc-plugin grpc-plugin}) + (verbose-prn "paths: %s" {:protoc protoc + :plugins (mapv :plugin-path plugins)}) (verbose-prn "output-path: %s" output-path) (doseq [[repo-id repo] repos-config] (let [repo-path (get repo-id->repo-path repo-id) proto-files (transduce - (map - ;; For backward compatibility, we allow either [[my_dir]] or [my_dir] - ;; as part of the `:dependencies` vector. - (fn [proto-dir-or-vec] - (let [proto-dir (if (vector? proto-dir-or-vec) - (first proto-dir-or-vec) - proto-dir-or-vec)] - (println "analyzing" proto-dir "... This may take a while for large repos") - (expand-dependencies - parallelism - protoc proto-paths - (discover-files repo-path (str proto-dir)))))) - sets/union - (:dependencies repo))] + (map + ;; For backward compatibility, we allow either [[my_dir]] or [my_dir] + ;; as part of the `:dependencies` vector. + (fn [proto-dir-or-vec] + (let [proto-dir (if (vector? proto-dir-or-vec) + (first proto-dir-or-vec) + proto-dir-or-vec)] + (println "analyzing" proto-dir "... This may take a while for large repos") + (expand-dependencies + parallelism + protoc proto-paths + (discover-files repo-path (str proto-dir)))))) + sets/union + (:dependencies repo))] (verbose-prn "files: %s" (mapv #(.getName ^File %) proto-files)) (when (empty? proto-files) (print-warning "could not find any .proto files under" repo-id)) (parallelize - parallelism - proto-files - (constantly nil) - (fn [proto-files] - (doseq [proto-file proto-files] - (let [protoc-opts (protoc-opts - proto-paths - output-path - (:compile-grpc? config) - grpc-plugin - proto-file)] - (println "compiling" (.getName proto-file)) - (run-protoc-and-report! protoc protoc-opts))))))) + parallelism + proto-files + (constantly nil) + (fn [proto-files] + (doseq [proto-file proto-files] + (let [protoc-opts-args (protoc-opts + proto-paths + output-path + plugins + proto-file)] + (println "compiling" (.getName proto-file)) + (run-protoc-and-report! protoc protoc-opts-args))))))) (finally (if keep-tmp? @@ -430,8 +543,8 @@ and include this full error message to add support for your platform." (defn generate-files*! "Generate protoc & gRPC stubs according to the `:lein-protodeps` configuration in `project.clj`" [opts project] - (let [config (:lein-protodeps project) - output-path (:output-path config)] + (let [config (:lein-protodeps project) + output-path (:output-path config)] (if (nil? config) (print-warning "No :lein-protodeps configuration found in project.clj") (binding [*verbose?* (-> opts :verbose)] @@ -454,15 +567,3 @@ and include this full error message to add support for your platform." (case mode "generate" (generate-files*! options project) (lein/warn "Unknown task" mode)))))) - -(comment - (def config '{:output-path "src/java/generated" - :proto-version "3.12.4" - :grpc-version "1.30.2" - :compile-grpc? true - :repos {:af-proto - {:repo-type :git - :proto-paths ["products"] - :config {:clone-url "git@localhost:test/repo.git" - :rev "mybranch"} - :dependencies [products/events]}}})) diff --git a/test/leiningen/protodeps_test.clj b/test/leiningen/protodeps_test.clj index ee22a2e..9f7b85a 100644 --- a/test/leiningen/protodeps_test.clj +++ b/test/leiningen/protodeps_test.clj @@ -73,9 +73,23 @@ set))))))) (deftest aarch64-architecture-mapping-test - (testing "Test that aarch64 architecture is correctly mapped to aarch_64 for protoc download URLs" - (is (= "aarch_64" (get sut/os-arch->arch "aarch64")) - "aarch64 should map to aarch_64 for correct protoc binary download URL"))) + (testing "Test that aarch64 architecture is correctly mapped with multiple variants" + (let [variants (get sut/os-arch->arch "aarch64")] + (is (= "aarch_64" (first variants)) + "aarch64 should map to aarch_64 as first variant (protoc style)") + (is (some #(= "arm64" %) variants) + "aarch64 should include arm64 as a variant (Go style)")))) + +(deftest platform-variants-test + (testing "Platform provides naming variants and stores all alternatives" + (let [env (doto (java.util.Properties.) + (.setProperty "os.name" "Mac OS X") + (.setProperty "os.arch" "aarch64")) + platform (@#'sut/get-platform env)] + (is (= "osx" (:os-name platform)) "Default os-name uses first variant (protoc style)") + (is (= "aarch_64" (:os-arch platform)) "Default os-arch uses first variant (protoc style)") + (is (= ["osx" "darwin"] (:os-name-variants platform)) "All OS name variants stored") + (is (= ["aarch_64" "arm64"] (:os-arch-variants platform)) "All arch variants stored")))) (deftest aarch64-url-generation-test (testing "Test that protoc download URL is correctly generated for aarch64 architecture" @@ -102,3 +116,98 @@ "Generated URL should match the correct format with aarch_64") (is (not= incorrect-url generated-url) "Generated URL should NOT match the incorrect format with aarch64")))) + +(deftest plugin-options-formatting-test + (testing "Format plugin options as key=value pairs" + (is (= "lang=java" (@#'sut/format-plugin-options {:lang "java"}))) + (is (= "lang=java,paths=source_relative" + (@#'sut/format-plugin-options {:lang "java" :paths "source_relative"}))) + (is (nil? (@#'sut/format-plugin-options {}))) + (is (nil? (@#'sut/format-plugin-options nil))))) + +(deftest plugin-url-interpolation-test + (testing "Plugin URL interpolation with platform and version variables" + (let [platform {:os-name "linux" :os-arch "x86_64" :version "1.0.2"} + template "https://example.com/releases/v${:version}/plugin-${:version}-${:os-name}-${:os-arch}"] + (is (= "https://example.com/releases/v1.0.2/plugin-1.0.2-linux-x86_64" + (@#'sut/interpolate platform template)))))) + +(deftest merge-legacy-grpc-config-test + (testing "Merge legacy gRPC config with new plugins" + (let [grpc-version {:semver "1.30.2" :os-name "linux" :os-arch "x86_64"}] + (testing "When compile-grpc? is true, add gRPC as a plugin" + (let [config {:compile-grpc? true :grpc-version "1.30.2"} + result (@#'sut/merge-legacy-grpc-config config grpc-version)] + (is (= 1 (count result))) + (is (= "protoc-gen-grpc-java" (:name (first result)))) + (is (= "1.30.2" (:version (first result)))) + (is (= "grpc-java_out" (:output-directive (first result)))))) + + (testing "When compile-grpc? is false, return only configured plugins" + (let [config {:compile-grpc? false + :plugins [{:name "protoc-gen-validate" + :version "1.0.2"}]} + result (@#'sut/merge-legacy-grpc-config config grpc-version)] + (is (= 1 (count result))) + (is (= "protoc-gen-validate" (:name (first result)))))) + + (testing "Merge gRPC with additional plugins" + (let [config {:compile-grpc? true + :grpc-version "1.30.2" + :plugins [{:name "protoc-gen-validate" + :version "1.0.2" + :url-template "https://example.com/validate" + :output-directive "validate_out"}]} + result (@#'sut/merge-legacy-grpc-config config grpc-version)] + (is (= 2 (count result))) + (is (= "protoc-gen-grpc-java" (:name (first result)))) + (is (= "protoc-gen-validate" (:name (second result)))))) + + (testing "No plugins when compile-grpc? is false and no plugins configured" + (let [config {:compile-grpc? false} + result (@#'sut/merge-legacy-grpc-config config grpc-version)] + (is (empty? result))))))) + +(deftest protoc-opts-with-plugins-test + (testing "Build protoc command with multiple plugins" + (let [proto-paths ["/path/to/protos"] + output-path "/output" + plugins [{:plugin-path "/plugins/protoc-gen-grpc-java" + :output-directive "grpc-java_out" + :options nil} + {:plugin-path "/plugins/protoc-gen-validate" + :output-directive "validate_out" + :options {:lang "java"}}] + proto-file (java.io.File. "/protos/test.proto") + result (@#'sut/protoc-opts proto-paths output-path plugins proto-file)] + (is (some #(= "--proto_path=/path/to/protos" %) result)) + (is (some #(= "--java_out=/output" %) result)) + (is (some #(= "--plugin=/plugins/protoc-gen-grpc-java" %) result)) + (is (some #(= "--grpc-java_out=/output" %) result)) + (is (some #(= "--plugin=/plugins/protoc-gen-validate" %) result)) + (is (some #(= "--validate_out=lang=java:/output" %) result)) + (is (some #(= "/protos/test.proto" %) result)))) + + (testing "Build protoc command without plugins" + (let [proto-paths ["/path/to/protos"] + output-path "/output" + plugins [] + proto-file (java.io.File. "/protos/test.proto") + result (@#'sut/protoc-opts proto-paths output-path plugins proto-file)] + (is (some #(= "--proto_path=/path/to/protos" %) result)) + (is (some #(= "--java_out=/output" %) result)) + (is (some #(= "/protos/test.proto" %) result)) + (is (not-any? #(.startsWith ^String % "--plugin=") result)))) + + (testing "Build protoc command with additional flags" + (let [proto-paths ["/path/to/protos"] + output-path "/output" + plugins [{:plugin-path "/plugins/protoc-gen-doc" + :output-directive "doc_out" + :options nil + :additional-flags {:doc_opt "markdown,docs.md"}}] + proto-file (java.io.File. "/protos/test.proto") + result (@#'sut/protoc-opts proto-paths output-path plugins proto-file)] + (is (some #(= "--plugin=/plugins/protoc-gen-doc" %) result)) + (is (some #(= "--doc_out=/output" %) result)) + (is (some #(= "--doc_opt=markdown,docs.md" %) result))))) From fe6cbf29f1799f839368d61312e479a3dc44235c Mon Sep 17 00:00:00 2001 From: Yevgeni Tsodikov Date: Tue, 9 Dec 2025 14:58:29 +0200 Subject: [PATCH 2/4] Add integration test for protoc-gen-validate plugin - Add test proto file with validation rules (user.proto) - Test that Validator files are NOT generated without the plugin - Test that Validator files ARE generated with the plugin - Use git dependency for validate.proto from protoc-gen-validate repo --- .../protos/validation/v1/user.proto | 20 +++++++ test/leiningen/protodeps_test.clj | 53 +++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 resources/test/proto_repo3/protos/validation/v1/user.proto diff --git a/resources/test/proto_repo3/protos/validation/v1/user.proto b/resources/test/proto_repo3/protos/validation/v1/user.proto new file mode 100644 index 0000000..b71f753 --- /dev/null +++ b/resources/test/proto_repo3/protos/validation/v1/user.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +package validation.v1; + +import "validate/validate.proto"; + +option java_package = "validation.v1"; +option java_multiple_files = true; + +message User { + // Email must be valid format + string email = 1 [(validate.rules).string.email = true]; + + // Age must be between 0 and 150 + int32 age = 2 [(validate.rules).int32 = {gte: 0, lte: 150}]; + + // Name must not be empty + string name = 3 [(validate.rules).string.min_len = 1]; +} + diff --git a/test/leiningen/protodeps_test.clj b/test/leiningen/protodeps_test.clj index 9f7b85a..66672a7 100644 --- a/test/leiningen/protodeps_test.clj +++ b/test/leiningen/protodeps_test.clj @@ -211,3 +211,56 @@ (is (some #(= "--plugin=/plugins/protoc-gen-doc" %) result)) (is (some #(= "--doc_out=/output" %) result)) (is (some #(= "--doc_opt=markdown,docs.md" %) result))))) + +(deftest protoc-gen-validate-plugin-test + (testing "Protoc-gen-validate plugin generates validator files" + (run-test! + (fn [tmp-dir-no-plugin] + ;; First, compile without the plugin - should NOT generate validator files + (let [config-no-plugin {:output-path (str tmp-dir-no-plugin) + :proto-version "3.25.5" + :repos '{:user-protos {:repo-type :filesystem + :config {:path "./resources/test/proto_repo3"} + :proto-paths ["protos"] + :dependencies [protos/validation]} + :validate {:repo-type :git + :config {:clone-url "https://github.com/bufbuild/protoc-gen-validate.git" + :rev "v1.3.0"} + :proto-paths ["."]}}}] + (sut/generate-files! {} config-no-plugin) + (let [files (->> (.toFile tmp-dir-no-plugin) + file-seq + (filter #(not (.isDirectory ^File %))) + (map #(.getName ^File %)) + set)] + (is (some #(= "User.java" %) files) "User.java should be generated") + (is (not (some #(.contains ^String % "Validator") files)) + "No Validator files should be generated without the plugin"))))) + + (run-test! + (fn [tmp-dir-with-plugin] + ;; Now compile WITH the plugin - SHOULD generate validator files + (let [config-with-plugin {:output-path (str tmp-dir-with-plugin) + :proto-version "3.25.5" + :plugins [{:name "protoc-gen-validate" + :version "1.3.0" + :url-template "https://github.com/bufbuild/protoc-gen-validate/releases/download/v${:version}/protoc-gen-validate_${:version}_${:os-name}_${:os-arch}.tar.gz" + :output-directive "validate_out" + :options {:lang "java"}}] + :repos '{:user-protos {:repo-type :filesystem + :config {:path "./resources/test/proto_repo3"} + :proto-paths ["protos"] + :dependencies [protos/validation]} + :validate {:repo-type :git + :config {:clone-url "https://github.com/bufbuild/protoc-gen-validate.git" + :rev "v1.3.0"} + :proto-paths ["."]}}}] + (sut/generate-files! {} config-with-plugin) + (let [files (->> (.toFile tmp-dir-with-plugin) + file-seq + (filter #(not (.isDirectory ^File %))) + (map #(.getName ^File %)) + set)] + (is (some #(= "User.java" %) files) "User.java should be generated") + (is (some #(= "UserValidator.java" %) files) + "UserValidator.java should be generated with the plugin"))))))) From 79acc1f4cb6f71bfc498a984f55120cc337e1cf4 Mon Sep 17 00:00:00 2001 From: Yevgeni Tsodikov Date: Tue, 9 Dec 2025 15:29:27 +0200 Subject: [PATCH 3/4] Fix gRPC Maven Central URL template separator Maven Central uses hyphens for all separators in gRPC binaries: - Wrong: protoc-gen-grpc-java-1.68.1-linux_x86_64.exe - Correct: protoc-gen-grpc-java-1.68.1-linux-x86_64.exe This fixes GitLab CI failures on Linux x86_64 --- src/leiningen/protodeps.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leiningen/protodeps.clj b/src/leiningen/protodeps.clj index 7517ff6..d8a0097 100644 --- a/src/leiningen/protodeps.clj +++ b/src/leiningen/protodeps.clj @@ -251,7 +251,7 @@ and include this full error message to add support for your platform." (def new-protoc-release-tpl "https://github.com/protocolbuffers/protobuf/releases/download/v${:minor}.${:patch}/protoc-${:minor}.${:patch}-${:os-name}-${:os-arch}.zip") -(def grpc-release-tpl "https://repo1.maven.org/maven2/io/grpc/protoc-gen-grpc-java/${:version}/protoc-gen-grpc-java-${:version}-${:os-name}_${:os-arch}.exe") +(def grpc-release-tpl "https://repo1.maven.org/maven2/io/grpc/protoc-gen-grpc-java/${:version}/protoc-gen-grpc-java-${:version}-${:os-name}-${:os-arch}.exe") (defn- protoc-release-template [{:keys [protoc-zip-url-template]} From 476e1e378c6f3926baf92f46754abd209affd585 Mon Sep 17 00:00:00 2001 From: Yevgeni Tsodikov Date: Wed, 10 Dec 2025 09:58:21 +0200 Subject: [PATCH 4/4] Refactor tests to use 'are' for better readability --- test/leiningen/protodeps_test.clj | 348 +++++++++++++++--------------- 1 file changed, 174 insertions(+), 174 deletions(-) diff --git a/test/leiningen/protodeps_test.clj b/test/leiningen/protodeps_test.clj index 66672a7..de9eb9b 100644 --- a/test/leiningen/protodeps_test.clj +++ b/test/leiningen/protodeps_test.clj @@ -1,8 +1,9 @@ (ns leiningen.protodeps-test - (:require [leiningen.protodeps :as sut] - [clojure.test :refer [deftest is testing]]) - (:import [java.nio.file Path] - [java.io File])) + (:require [clojure.test :refer [are deftest is testing]] + [leiningen.protodeps :as sut]) + (:import (java.io File) + (java.nio.file Path) + (java.util Properties))) (defn- run-test! [test] (let [^Path tmp-dir (sut/create-temp-dir!)] @@ -13,64 +14,64 @@ (deftest integration-test (run-test! - (fn [tmp-dir] - (let [config {:output-path (str tmp-dir) - :proto-version "3.11.3" - :repos '{:repo1 {:repo-type :filesystem - :config {:path "./resources/test/proto_repo"} - :proto-paths ["protos"] - :dependencies [protos]} - ;; external dependency repo, no direct schemas to compile - :repo2 {:repo-type :filesystem - :config {:path "./resources/test/proto_repo2"} - :proto-paths ["protos"]}}}] - (sut/generate-files! {} config) - (is (= #{"dir1/v1/File1.java" "dir2/v1/File2.java" "dir3/v1/File3.java" "dir4/v1/File4.java"} - (->> (.toFile tmp-dir) - file-seq - (filter #(not (.isDirectory ^File %))) - (map #(.relativize tmp-dir (.toPath ^File %))) - (map str) - set)))))) + (fn [tmp-dir] + (let [config {:output-path (str tmp-dir) + :proto-version "3.11.3" + :repos '{:repo1 {:repo-type :filesystem + :config {:path "./resources/test/proto_repo"} + :proto-paths ["protos"] + :dependencies [protos]} + ;; external dependency repo, no direct schemas to compile + :repo2 {:repo-type :filesystem + :config {:path "./resources/test/proto_repo2"} + :proto-paths ["protos"]}}}] + (sut/generate-files! {} config) + (is (= #{"dir1/v1/File1.java" "dir2/v1/File2.java" "dir3/v1/File3.java" "dir4/v1/File4.java"} + (->> (.toFile tmp-dir) + file-seq + (filter #(not (.isDirectory ^File %))) + (map #(.relativize tmp-dir (.toPath ^File %))) + (map str) + set)))))) (run-test! - (fn [tmp-dir] - (let [config {:output-path (str tmp-dir) - :proto-version "3.11.3" - :repos '{:repo1 {:repo-type :filesystem - :config {:path "./resources/test/proto_repo"} - :proto-paths ["protos"] - :dependencies [protos/dir1]} - ;; external dependency repo, no direct schemas to compile - :repo2 {:repo-type :filesystem - :config {:path "./resources/test/proto_repo2"} - :proto-paths ["protos"]}}}] - (sut/generate-files! {} config) - (is (= #{"dir1/v1/File1.java" "dir2/v1/File2.java" "dir3/v1/File3.java"} - (->> (.toFile tmp-dir) - file-seq - (filter #(not (.isDirectory ^File %))) - (map #(.relativize tmp-dir (.toPath ^File %))) - (map str) - set)))))) + (fn [tmp-dir] + (let [config {:output-path (str tmp-dir) + :proto-version "3.11.3" + :repos '{:repo1 {:repo-type :filesystem + :config {:path "./resources/test/proto_repo"} + :proto-paths ["protos"] + :dependencies [protos/dir1]} + ;; external dependency repo, no direct schemas to compile + :repo2 {:repo-type :filesystem + :config {:path "./resources/test/proto_repo2"} + :proto-paths ["protos"]}}}] + (sut/generate-files! {} config) + (is (= #{"dir1/v1/File1.java" "dir2/v1/File2.java" "dir3/v1/File3.java"} + (->> (.toFile tmp-dir) + file-seq + (filter #(not (.isDirectory ^File %))) + (map #(.relativize tmp-dir (.toPath ^File %))) + (map str) + set)))))) (run-test! - (fn [tmp-dir] - (let [config {:output-path (str tmp-dir) - :proto-version "3.11.3" - :repos '{:repo1 {:repo-type :filesystem - :config {:path "./resources/test/proto_repo"} - :proto-paths ["protos"]} - :repo2 {:repo-type :filesystem - :config {:path "./resources/test/proto_repo2"} - :proto-paths ["protos"] - :dependencies [protos/dir3]}}}] - (sut/generate-files! {} config) - (is (= #{"dir3/v1/File3.java"} - (->> (.toFile tmp-dir) - file-seq - (filter #(not (.isDirectory ^File %))) - (map #(.relativize tmp-dir (.toPath ^File %))) - (map str) - set))))))) + (fn [tmp-dir] + (let [config {:output-path (str tmp-dir) + :proto-version "3.11.3" + :repos '{:repo1 {:repo-type :filesystem + :config {:path "./resources/test/proto_repo"} + :proto-paths ["protos"]} + :repo2 {:repo-type :filesystem + :config {:path "./resources/test/proto_repo2"} + :proto-paths ["protos"] + :dependencies [protos/dir3]}}}] + (sut/generate-files! {} config) + (is (= #{"dir3/v1/File3.java"} + (->> (.toFile tmp-dir) + file-seq + (filter #(not (.isDirectory ^File %))) + (map #(.relativize tmp-dir (.toPath ^File %))) + (map str) + set))))))) (deftest aarch64-architecture-mapping-test (testing "Test that aarch64 architecture is correctly mapped with multiple variants" @@ -82,9 +83,9 @@ (deftest platform-variants-test (testing "Platform provides naming variants and stores all alternatives" - (let [env (doto (java.util.Properties.) - (.setProperty "os.name" "Mac OS X") - (.setProperty "os.arch" "aarch64")) + (let [env (doto (Properties.) + (.setProperty "os.name" "Mac OS X") + (.setProperty "os.arch" "aarch64")) platform (@#'sut/get-platform env)] (is (= "osx" (:os-name platform)) "Default os-name uses first variant (protoc style)") (is (= "aarch_64" (:os-arch platform)) "Default os-arch uses first variant (protoc style)") @@ -93,18 +94,18 @@ (deftest aarch64-url-generation-test (testing "Test that protoc download URL is correctly generated for aarch64 architecture" - (let [platform {:os-name "linux" - :os-arch "aarch_64" - :semver "24.3"} - url-template "https://github.com/protocolbuffers/protobuf/releases/download/v${:semver}/protoc-${:semver}-${:os-name}-${:os-arch}.zip" - expected-url "https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-aarch_64.zip"] + (let [platform {:os-name "linux" + :os-arch "aarch_64" + :semver "24.3"} + url-template "https://github.com/protocolbuffers/protobuf/releases/download/v${:semver}/protoc-${:semver}-${:os-name}-${:os-arch}.zip" + expected-url "https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-aarch_64.zip"] (is (= expected-url (@#'sut/interpolate platform url-template)) "URL should be correctly generated with aarch_64 architecture name")))) (deftest aarch64-issue-8-fix-test (testing "Test that the fix for GitHub issue #8 works correctly" (let [platform {:os-name "linux" - :os-arch "aarch_64" + :os-arch "aarch_64" :semver "24.3"} url-template "https://github.com/protocolbuffers/protobuf/releases/download/v${:semver}/protoc-${:semver}-${:os-name}-${:os-arch}.zip" generated-url (@#'sut/interpolate platform url-template) @@ -119,11 +120,11 @@ (deftest plugin-options-formatting-test (testing "Format plugin options as key=value pairs" - (is (= "lang=java" (@#'sut/format-plugin-options {:lang "java"}))) - (is (= "lang=java,paths=source_relative" - (@#'sut/format-plugin-options {:lang "java" :paths "source_relative"}))) - (is (nil? (@#'sut/format-plugin-options {}))) - (is (nil? (@#'sut/format-plugin-options nil))))) + (are [expected input] (= expected (@#'sut/format-plugin-options input)) + "lang=java" {:lang "java"} + "lang=java,paths=source_relative" {:lang "java" :paths "source_relative"} + nil {} + nil nil))) (deftest plugin-url-interpolation-test (testing "Plugin URL interpolation with platform and version variables" @@ -142,125 +143,124 @@ (is (= "protoc-gen-grpc-java" (:name (first result)))) (is (= "1.30.2" (:version (first result)))) (is (= "grpc-java_out" (:output-directive (first result)))))) - + (testing "When compile-grpc? is false, return only configured plugins" (let [config {:compile-grpc? false - :plugins [{:name "protoc-gen-validate" - :version "1.0.2"}]} + :plugins [{:name "protoc-gen-validate" + :version "1.0.2"}]} result (@#'sut/merge-legacy-grpc-config config grpc-version)] (is (= 1 (count result))) (is (= "protoc-gen-validate" (:name (first result)))))) - + (testing "Merge gRPC with additional plugins" (let [config {:compile-grpc? true - :grpc-version "1.30.2" - :plugins [{:name "protoc-gen-validate" - :version "1.0.2" - :url-template "https://example.com/validate" - :output-directive "validate_out"}]} + :grpc-version "1.30.2" + :plugins [{:name "protoc-gen-validate" + :version "1.0.2" + :url-template "https://example.com/validate" + :output-directive "validate_out"}]} result (@#'sut/merge-legacy-grpc-config config grpc-version)] (is (= 2 (count result))) (is (= "protoc-gen-grpc-java" (:name (first result)))) (is (= "protoc-gen-validate" (:name (second result)))))) - + (testing "No plugins when compile-grpc? is false and no plugins configured" (let [config {:compile-grpc? false} result (@#'sut/merge-legacy-grpc-config config grpc-version)] (is (empty? result))))))) (deftest protoc-opts-with-plugins-test - (testing "Build protoc command with multiple plugins" - (let [proto-paths ["/path/to/protos"] - output-path "/output" - plugins [{:plugin-path "/plugins/protoc-gen-grpc-java" - :output-directive "grpc-java_out" - :options nil} - {:plugin-path "/plugins/protoc-gen-validate" - :output-directive "validate_out" - :options {:lang "java"}}] - proto-file (java.io.File. "/protos/test.proto") - result (@#'sut/protoc-opts proto-paths output-path plugins proto-file)] - (is (some #(= "--proto_path=/path/to/protos" %) result)) - (is (some #(= "--java_out=/output" %) result)) - (is (some #(= "--plugin=/plugins/protoc-gen-grpc-java" %) result)) - (is (some #(= "--grpc-java_out=/output" %) result)) - (is (some #(= "--plugin=/plugins/protoc-gen-validate" %) result)) - (is (some #(= "--validate_out=lang=java:/output" %) result)) - (is (some #(= "/protos/test.proto" %) result)))) - - (testing "Build protoc command without plugins" - (let [proto-paths ["/path/to/protos"] - output-path "/output" - plugins [] - proto-file (java.io.File. "/protos/test.proto") - result (@#'sut/protoc-opts proto-paths output-path plugins proto-file)] - (is (some #(= "--proto_path=/path/to/protos" %) result)) - (is (some #(= "--java_out=/output" %) result)) - (is (some #(= "/protos/test.proto" %) result)) - (is (not-any? #(.startsWith ^String % "--plugin=") result)))) - - (testing "Build protoc command with additional flags" - (let [proto-paths ["/path/to/protos"] - output-path "/output" - plugins [{:plugin-path "/plugins/protoc-gen-doc" - :output-directive "doc_out" - :options nil - :additional-flags {:doc_opt "markdown,docs.md"}}] - proto-file (java.io.File. "/protos/test.proto") - result (@#'sut/protoc-opts proto-paths output-path plugins proto-file)] - (is (some #(= "--plugin=/plugins/protoc-gen-doc" %) result)) - (is (some #(= "--doc_out=/output" %) result)) - (is (some #(= "--doc_opt=markdown,docs.md" %) result))))) + (let [proto-paths ["/path/to/protos"] + output-path "/output" + proto-file (File. "/protos/test.proto") + contains-flag? (fn [result flag] (some #(= flag %) result))] + + (testing "Build protoc command with multiple plugins" + (let [plugins [{:plugin-path "/plugins/protoc-gen-grpc-java" + :output-directive "grpc-java_out" + :options nil} + {:plugin-path "/plugins/protoc-gen-validate" + :output-directive "validate_out" + :options {:lang "java"}}] + result (@#'sut/protoc-opts proto-paths output-path plugins proto-file)] + (are [flag] (contains-flag? result flag) + "--proto_path=/path/to/protos" + "--java_out=/output" + "--plugin=/plugins/protoc-gen-grpc-java" + "--grpc-java_out=/output" + "--plugin=/plugins/protoc-gen-validate" + "--validate_out=lang=java:/output" + "/protos/test.proto"))) + + (testing "Build protoc command without plugins" + (let [plugins [] + result (@#'sut/protoc-opts proto-paths output-path plugins proto-file)] + (are [flag] (contains-flag? result flag) + "--proto_path=/path/to/protos" + "--java_out=/output" + "/protos/test.proto") + (is (not-any? #(.startsWith ^String % "--plugin=") result)))) + + (testing "Build protoc command with additional flags" + (let [plugins [{:plugin-path "/plugins/protoc-gen-doc" + :output-directive "doc_out" + :options nil + :additional-flags {:doc_opt "markdown,docs.md"}}] + result (@#'sut/protoc-opts proto-paths output-path plugins proto-file)] + (are [flag] (contains-flag? result flag) + "--plugin=/plugins/protoc-gen-doc" + "--doc_out=/output" + "--doc_opt=markdown,docs.md"))))) (deftest protoc-gen-validate-plugin-test (testing "Protoc-gen-validate plugin generates validator files" (run-test! - (fn [tmp-dir-no-plugin] - ;; First, compile without the plugin - should NOT generate validator files - (let [config-no-plugin {:output-path (str tmp-dir-no-plugin) - :proto-version "3.25.5" - :repos '{:user-protos {:repo-type :filesystem - :config {:path "./resources/test/proto_repo3"} - :proto-paths ["protos"] - :dependencies [protos/validation]} - :validate {:repo-type :git - :config {:clone-url "https://github.com/bufbuild/protoc-gen-validate.git" - :rev "v1.3.0"} - :proto-paths ["."]}}}] - (sut/generate-files! {} config-no-plugin) - (let [files (->> (.toFile tmp-dir-no-plugin) - file-seq - (filter #(not (.isDirectory ^File %))) - (map #(.getName ^File %)) - set)] - (is (some #(= "User.java" %) files) "User.java should be generated") - (is (not (some #(.contains ^String % "Validator") files)) - "No Validator files should be generated without the plugin"))))) - + (fn [tmp-dir-no-plugin] + ;; First, compile without the plugin - should NOT generate validator files + (let [config-no-plugin {:output-path (str tmp-dir-no-plugin) + :proto-version "3.25.5" + :repos '{:user-protos {:repo-type :filesystem + :config {:path "./resources/test/proto_repo3"} + :proto-paths ["protos"] + :dependencies [protos/validation]} + :validate {:repo-type :git + :config {:clone-url "https://github.com/bufbuild/protoc-gen-validate.git" + :rev "v1.3.0"} + :proto-paths ["."]}}}] + (sut/generate-files! {} config-no-plugin) + (let [files (->> (.toFile tmp-dir-no-plugin) + file-seq + (filter #(not (.isDirectory ^File %))) + (map #(.getName ^File %)) + set)] + (is (some #(= "User.java" %) files) "User.java should be generated") + (is (not (some #(.contains ^String % "Validator") files)) + "No Validator files should be generated without the plugin"))))) + (run-test! - (fn [tmp-dir-with-plugin] - ;; Now compile WITH the plugin - SHOULD generate validator files - (let [config-with-plugin {:output-path (str tmp-dir-with-plugin) - :proto-version "3.25.5" - :plugins [{:name "protoc-gen-validate" - :version "1.3.0" - :url-template "https://github.com/bufbuild/protoc-gen-validate/releases/download/v${:version}/protoc-gen-validate_${:version}_${:os-name}_${:os-arch}.tar.gz" - :output-directive "validate_out" - :options {:lang "java"}}] - :repos '{:user-protos {:repo-type :filesystem - :config {:path "./resources/test/proto_repo3"} - :proto-paths ["protos"] - :dependencies [protos/validation]} - :validate {:repo-type :git - :config {:clone-url "https://github.com/bufbuild/protoc-gen-validate.git" - :rev "v1.3.0"} - :proto-paths ["."]}}}] - (sut/generate-files! {} config-with-plugin) - (let [files (->> (.toFile tmp-dir-with-plugin) - file-seq - (filter #(not (.isDirectory ^File %))) - (map #(.getName ^File %)) - set)] - (is (some #(= "User.java" %) files) "User.java should be generated") - (is (some #(= "UserValidator.java" %) files) - "UserValidator.java should be generated with the plugin"))))))) + (fn [tmp-dir-with-plugin] + ;; Now compile WITH the plugin - SHOULD generate validator files + (let [config-with-plugin {:output-path (str tmp-dir-with-plugin) + :proto-version "3.25.5" + :plugins [{:name "protoc-gen-validate" + :version "1.3.0" + :url-template "https://github.com/bufbuild/protoc-gen-validate/releases/download/v${:version}/protoc-gen-validate_${:version}_${:os-name}_${:os-arch}.tar.gz" + :output-directive "validate_out" + :options {:lang "java"}}] + :repos '{:user-protos {:repo-type :filesystem + :config {:path "./resources/test/proto_repo3"} + :proto-paths ["protos"] + :dependencies [protos/validation]} + :validate {:repo-type :git + :config {:clone-url "https://github.com/bufbuild/protoc-gen-validate.git" + :rev "v1.3.0"} + :proto-paths ["."]}}}] + (sut/generate-files! {} config-with-plugin) + (let [files (->> (.toFile tmp-dir-with-plugin) + file-seq + (filter #(not (.isDirectory ^File %))) + (map #(.getName ^File %)) + set)] + (is (some #(= "User.java" %) files) "User.java should be generated") + (is (some #(= "UserValidator.java" %) files) + "UserValidator.java should be generated with the plugin")))))))