From f7be272e996a44e0af1fab6dbadef13c4dd04241 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 01:16:13 -0400 Subject: [PATCH 01/22] feat(agent): add the agent core that keeps policies current in systems grantforge-agent-core runs on Java 8 inside the systems GrantForge protects. GrantForgeAgent sends heartbeats, downloads the service's snapshot when the policy version changes, checks its Ed25519 signature with Bouncy Castle, builds a policy engine from it with Jackson 2 and keeps the last good snapshot on disk for starts without the server. Decisions add the roles and groups the snapshot gives the user; access events go out in batches and are spooled to disk while the server is away. --- .github/workflows/ci.yml | 6 +- core/grantforge-agent-core/pom.xml | 66 +++ .../devlive/grantforge/agent/AccessEvent.java | 213 ++++++++ .../grantforge/agent/AgentDecision.java | 118 +++++ .../grantforge/agent/AgentSettings.java | 438 ++++++++++++++++ .../grantforge/agent/AuditShipper.java | 219 ++++++++ .../grantforge/agent/GrantForgeAgent.java | 313 +++++++++++ .../grantforge/agent/ServerClient.java | 289 +++++++++++ .../devlive/grantforge/agent/SigningKey.java | 122 +++++ .../devlive/grantforge/agent/Snapshot.java | 484 ++++++++++++++++++ .../grantforge/agent/SnapshotStore.java | 133 +++++ .../grantforge/agent/package-info.java | 21 + .../grantforge/agent/AccessEventTest.java | 47 ++ .../grantforge/agent/AgentDecisionTest.java | 39 ++ .../grantforge/agent/AgentSettingsTest.java | 87 ++++ .../grantforge/agent/AuditShipperTest.java | 127 +++++ .../devlive/grantforge/agent/FakeServer.java | 245 +++++++++ .../grantforge/agent/GrantForgeAgentTest.java | 200 ++++++++ .../grantforge/agent/ServerClientTest.java | 132 +++++ .../grantforge/agent/SigningKeyTest.java | 57 +++ .../grantforge/agent/SnapshotStoreTest.java | 79 +++ .../grantforge/agent/SnapshotTest.java | 153 ++++++ .../devlive/grantforge/agent/Snapshots.java | 74 +++ docs/content/architecture/plugins.md | 20 + pom.xml | 6 + script/ci/coverage_thresholds.txt | 1 + 26 files changed, 3687 insertions(+), 2 deletions(-) create mode 100644 core/grantforge-agent-core/pom.xml create mode 100644 core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AccessEvent.java create mode 100644 core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AgentDecision.java create mode 100644 core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AgentSettings.java create mode 100644 core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AuditShipper.java create mode 100644 core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/GrantForgeAgent.java create mode 100644 core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/ServerClient.java create mode 100644 core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/SigningKey.java create mode 100644 core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/Snapshot.java create mode 100644 core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/SnapshotStore.java create mode 100644 core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/package-info.java create mode 100644 core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AccessEventTest.java create mode 100644 core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AgentDecisionTest.java create mode 100644 core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AgentSettingsTest.java create mode 100644 core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AuditShipperTest.java create mode 100644 core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/FakeServer.java create mode 100644 core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/GrantForgeAgentTest.java create mode 100644 core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/ServerClientTest.java create mode 100644 core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/SigningKeyTest.java create mode 100644 core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/SnapshotStoreTest.java create mode 100644 core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/SnapshotTest.java create mode 100644 core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/Snapshots.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0cb877b0..a43cc9cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,8 +103,10 @@ jobs: run: bash script/ci/java.sh test - name: Check Java 17 bytecode compatibility run: python3 script/ci/check_java_bytecode.py --max-major 61 - - name: Check Java 8 bytecode of the policy engine agents embed - run: python3 script/ci/check_java_bytecode.py --max-major 52 --module core/grantforge-policy-engine --main-only + - name: Check Java 8 bytecode of the policy engine and agent core agents embed + run: | + python3 script/ci/check_java_bytecode.py --max-major 52 --module core/grantforge-policy-engine --main-only + python3 script/ci/check_java_bytecode.py --max-major 52 --module core/grantforge-agent-core --main-only - name: Upload test reports if: always() uses: actions/upload-artifact@v4 diff --git a/core/grantforge-agent-core/pom.xml b/core/grantforge-agent-core/pom.xml new file mode 100644 index 00000000..72d81eb5 --- /dev/null +++ b/core/grantforge-agent-core/pom.xml @@ -0,0 +1,66 @@ + + + + + + grantforge + org.devlive.grantforge + 2026.0.0 + ../../pom.xml + + 4.0.0 + + grantforge-agent-core + GrantForge Agent Core + What every agent in a protected system shares: downloads and checks the signed policy snapshots of its + service, keeps the last one on disk, decides access with the policy engine and ships access events; runs on + Java 8 with the engine, Jackson 2 and Bouncy Castle + + + + 8 + 17 + + + + + org.devlive.grantforge + grantforge-policy-engine + + + + com.fasterxml.jackson.core + jackson-databind + + + + org.bouncycastle + bcprov-jdk18on + + + + org.jspecify + jspecify + provided + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.devlive.grantforge + grantforge-test-support + test + + + diff --git a/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AccessEvent.java b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AccessEvent.java new file mode 100644 index 00000000..e959941f --- /dev/null +++ b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AccessEvent.java @@ -0,0 +1,213 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import org.jspecify.annotations.Nullable; + +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; + +/** + * One access the agent saw, as the server's access audit stores it. Built with {@link #builder}; each event gets its + * own id, which makes sending it again harmless. Immutable. + */ +public final class AccessEvent +{ + /** Longest request text kept, as the server stores it. */ + public static final int MAX_REQUEST = 1000; + + private final String eventId; + private final Instant occurredAt; + private final String user; + private final String resource; + private final String accessType; + private final boolean allowed; + private final @Nullable String clientIp; + private final @Nullable String resourceType; + private final @Nullable String action; + private final @Nullable Long policyId; + private final @Nullable Long policyVersion; + private final boolean byGrantForge; + private final @Nullable String request; + + AccessEvent(Builder builder) + { + this.eventId = UUID.randomUUID().toString(); + this.occurredAt = builder.occurredAt; + this.user = builder.user; + this.resource = builder.resource; + this.accessType = builder.accessType; + this.allowed = builder.allowed; + this.clientIp = builder.clientIp; + this.resourceType = builder.resourceType; + this.action = builder.action; + this.policyId = builder.policyId; + this.policyVersion = builder.policyVersion; + this.byGrantForge = builder.byGrantForge; + String text = builder.request; + this.request = text == null || text.length() <= MAX_REQUEST ? text : text.substring(0, MAX_REQUEST); + } + + /** + * Starts an event. + * + * @param user who + * @param resource the resource, its levels joined as the system shows them, such as {@code /data/sales} or + * {@code sales.orders.ssn} + * @param accessType the access type checked + * @param allowed whether access was allowed in the end + * @return a builder + */ + public static Builder builder(String user, String resource, String accessType, boolean allowed) + { + return new Builder(user, resource, accessType, allowed); + } + + /** + * Returns the event's id. + * + * @return a UUID + */ + public String eventId() + { + return eventId; + } + + /** + * Returns the event as the server's access event API takes it. + * + * @return the fields, by name + */ + Map fields() + { + Map fields = new LinkedHashMap<>(); + fields.put("eventId", eventId); + fields.put("occurredAt", occurredAt.toString()); + fields.put("user", user); + fields.put("clientIp", clientIp); + fields.put("resource", resource); + fields.put("resourceType", resourceType); + fields.put("accessType", accessType); + fields.put("action", action); + fields.put("outcome", allowed ? "ALLOWED" : "DENIED"); + fields.put("policyId", policyId == null ? null : Long.toString(policyId)); + fields.put("policyVersion", policyVersion); + fields.put("enforcer", byGrantForge ? "GRANTFORGE" : "NATIVE"); + fields.put("request", request); + return fields; + } + + /** Collects an event. */ + public static final class Builder + { + final String user; + final String resource; + final String accessType; + final boolean allowed; + Instant occurredAt = Instant.now(); + @Nullable String clientIp; + @Nullable String resourceType; + @Nullable String action; + @Nullable Long policyId; + @Nullable Long policyVersion; + boolean byGrantForge; + @Nullable String request; + + Builder(String user, String resource, String accessType, boolean allowed) + { + this.user = user; + this.resource = resource; + this.accessType = accessType; + this.allowed = allowed; + } + + /** + * Says who decided: GrantForge when the decision was determined, the system's own checks otherwise. + * + * @param decision the agent's decision + * @return this builder + */ + public Builder decidedBy(AgentDecision decision) + { + this.byGrantForge = decision.determined(); + this.policyId = decision.policyId(); + this.policyVersion = decision.policyVersion(); + return this; + } + + /** + * Sets when it happened; now by default. + * + * @param value the moment + * @return this builder + */ + public Builder occurredAt(Instant value) + { + this.occurredAt = value; + return this; + } + + /** + * Sets from where. + * + * @param value the client address + * @return this builder + */ + public Builder clientIp(@Nullable String value) + { + this.clientIp = value; + return this; + } + + /** + * Sets the lowest level of the resource, such as {@code column}. + * + * @param value the level + * @return this builder + */ + public Builder resourceType(@Nullable String value) + { + this.resourceType = value; + return this; + } + + /** + * Sets the system's operation, such as {@code open} or {@code SELECT}. + * + * @param value the operation + * @return this builder + */ + public Builder action(@Nullable String value) + { + this.action = value; + return this; + } + + /** + * Sets the request, such as an SQL statement; cut to {@value #MAX_REQUEST} characters. + * + * @param value the request + * @return this builder + */ + public Builder request(@Nullable String value) + { + this.request = value; + return this; + } + + /** + * Builds the event. + * + * @return the event + */ + public AccessEvent build() + { + return new AccessEvent(this); + } + } +} diff --git a/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AgentDecision.java b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AgentDecision.java new file mode 100644 index 00000000..0adc7c1d --- /dev/null +++ b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AgentDecision.java @@ -0,0 +1,118 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import org.jspecify.annotations.Nullable; + +/** + * What the agent decided about a request, with the policy and the policy version behind it, as access events report + * them. Immutable. + */ +public final class AgentDecision +{ + /** How a request was decided. */ + public enum Outcome + { + /** A policy allowed it. */ + ALLOWED, + /** A policy denied it. */ + DENIED, + /** + * No policy decided, the service is not in use or the agent has no snapshot yet; the system's own checks apply, + * or access is denied where the agent is configured so. + */ + NOT_DETERMINED + } + + private static final AgentDecision WITHOUT_SNAPSHOT = new AgentDecision(Outcome.NOT_DETERMINED, null, null); + + private final Outcome outcome; + private final @Nullable Long policyId; + private final @Nullable Long policyVersion; + + private AgentDecision(Outcome outcome, @Nullable Long policyId, @Nullable Long policyVersion) + { + this.outcome = outcome; + this.policyId = policyId; + this.policyVersion = policyVersion; + } + + static AgentDecision allowed(long policyVersion, long policyId) + { + return new AgentDecision(Outcome.ALLOWED, policyId, policyVersion); + } + + static AgentDecision denied(long policyVersion, long policyId) + { + return new AgentDecision(Outcome.DENIED, policyId, policyVersion); + } + + static AgentDecision notDetermined(long policyVersion) + { + return new AgentDecision(Outcome.NOT_DETERMINED, null, policyVersion); + } + + static AgentDecision withoutSnapshot() + { + return WITHOUT_SNAPSHOT; + } + + /** + * Returns how the request was decided. + * + * @return the outcome + */ + public Outcome outcome() + { + return outcome; + } + + /** + * Returns whether a policy allowed the request. + * + * @return {@code true} if one did + */ + public boolean allowed() + { + return outcome == Outcome.ALLOWED; + } + + /** + * Returns whether GrantForge decided, rather than leaving it to the system. + * + * @return {@code true} if a policy allowed or denied the request + */ + public boolean determined() + { + return outcome != Outcome.NOT_DETERMINED; + } + + /** + * Returns the policy that decided. + * + * @return its id, or {@code null} if none did + */ + public @Nullable Long policyId() + { + return policyId; + } + + /** + * Returns the policy version the decision was made with. + * + * @return the version, or {@code null} without a snapshot + */ + public @Nullable Long policyVersion() + { + return policyVersion; + } + + @Override + public String toString() + { + return outcome + (policyId == null ? "" : " by policy " + policyId) + (policyVersion == null ? "" : " at version " + policyVersion); + } +} diff --git a/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AgentSettings.java b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AgentSettings.java new file mode 100644 index 00000000..8afdd945 --- /dev/null +++ b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AgentSettings.java @@ -0,0 +1,438 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import org.jspecify.annotations.Nullable; + +import java.net.InetAddress; +import java.net.URI; +import java.net.UnknownHostException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.regex.Pattern; + +/** + * How an agent reaches its server and handles policies and events; built with {@link #builder}, validated when built. + * Immutable. + */ +public final class AgentSettings +{ + /** The most events the server takes in one batch. */ + public static final int MAX_BATCH = 1000; + + static final Pattern INSTANCE = Pattern.compile("\\p{Graph}{1,128}"); + + private final URI server; + private final String token; + private final String instance; + private final String host; + private final String agentVersion; + private final Path cacheDirectory; + private final Duration connectTimeout; + private final Duration readTimeout; + private final Duration refreshInterval; + private final int auditBatchSize; + private final Duration auditFlushInterval; + private final int auditQueueCapacity; + private final long spoolLimitBytes; + private final @Nullable SigningKey trustedKey; + + AgentSettings(Builder builder, URI server, String token, String instance, Path cacheDirectory) + { + this.server = server; + this.token = token; + this.instance = instance; + this.host = builder.host == null ? localHost() : builder.host; + this.agentVersion = builder.agentVersion; + this.cacheDirectory = cacheDirectory; + this.connectTimeout = builder.connectTimeout; + this.readTimeout = builder.readTimeout; + this.refreshInterval = builder.refreshInterval; + this.auditBatchSize = builder.auditBatchSize; + this.auditFlushInterval = builder.auditFlushInterval; + this.auditQueueCapacity = builder.auditQueueCapacity; + this.spoolLimitBytes = builder.spoolLimitBytes; + this.trustedKey = builder.trustedKey; + } + + /** + * Starts settings. + * + * @return a builder + */ + public static Builder builder() + { + return new Builder(); + } + + private static String localHost() + { + try { + return InetAddress.getLocalHost().getHostName(); + } + catch (UnknownHostException unknown) { + return "unknown"; + } + } + + /** + * Returns the server's address, such as {@code https://grantforge.example.com/}. + * + * @return the address + */ + public URI server() + { + return server; + } + + /** + * Returns the service's agent token, as the console issues it. + * + * @return the token + */ + public String token() + { + return token; + } + + /** + * Returns the name this agent gives itself, unique within its service, such as {@code namenode-1:8020}. + * + * @return the name + */ + public String instance() + { + return instance; + } + + /** + * Returns where the agent runs. + * + * @return the host name + */ + public String host() + { + return host; + } + + /** + * Returns the agent's version, as heartbeats report it. + * + * @return the version + */ + public String agentVersion() + { + return agentVersion; + } + + /** + * Returns where the last snapshot and the events not yet sent are kept. + * + * @return the directory + */ + public Path cacheDirectory() + { + return cacheDirectory; + } + + /** + * Returns how long to wait for a connection to the server. + * + * @return the time + */ + public Duration connectTimeout() + { + return connectTimeout; + } + + /** + * Returns how long to wait for the server's answer. + * + * @return the time + */ + public Duration readTimeout() + { + return readTimeout; + } + + /** + * Returns the time between heartbeats until the server sets it, and while it cannot be reached. + * + * @return the time + */ + public Duration refreshInterval() + { + return refreshInterval; + } + + /** + * Returns the most events sent in one batch. + * + * @return the count, at most {@value #MAX_BATCH} + */ + public int auditBatchSize() + { + return auditBatchSize; + } + + /** + * Returns how often events are sent when there are fewer than a batch. + * + * @return the time + */ + public Duration auditFlushInterval() + { + return auditFlushInterval; + } + + /** + * Returns how many events wait in memory; beyond that, recording drops events rather than slow the system down. + * + * @return the count + */ + public int auditQueueCapacity() + { + return auditQueueCapacity; + } + + /** + * Returns how much disk the events that could not be sent may take; beyond that the oldest are dropped. + * + * @return the size in bytes + */ + public long spoolLimitBytes() + { + return spoolLimitBytes; + } + + /** + * Returns the server's public key, when it is given rather than fetched from the server on first contact. + * + * @return the key, or {@code null} + */ + public @Nullable SigningKey trustedKey() + { + return trustedKey; + } + + /** Collects settings. */ + public static final class Builder + { + @Nullable URI server; + @Nullable String token; + @Nullable String instance; + @Nullable String host; + String agentVersion = "unknown"; + @Nullable Path cacheDirectory; + Duration connectTimeout = Duration.ofSeconds(10); + Duration readTimeout = Duration.ofSeconds(30); + Duration refreshInterval = Duration.ofSeconds(30); + int auditBatchSize = 500; + Duration auditFlushInterval = Duration.ofSeconds(5); + int auditQueueCapacity = 10_000; + long spoolLimitBytes = 64L * 1024 * 1024; + @Nullable SigningKey trustedKey; + + Builder() + { + } + + /** + * Sets the server's address. + * + * @param value an http or https address + * @return this builder + */ + public Builder server(URI value) + { + this.server = value; + return this; + } + + /** + * Sets the agent token. + * + * @param value the token + * @return this builder + */ + public Builder token(String value) + { + this.token = value; + return this; + } + + /** + * Sets the name the agent gives itself. + * + * @param value 1-128 visible characters, unique within the service + * @return this builder + */ + public Builder instance(String value) + { + this.instance = value; + return this; + } + + /** + * Sets where the agent runs; the local host name by default. + * + * @param value the host + * @return this builder + */ + public Builder host(String value) + { + this.host = value; + return this; + } + + /** + * Sets the agent's version. + * + * @param value the version + * @return this builder + */ + public Builder agentVersion(String value) + { + this.agentVersion = value; + return this; + } + + /** + * Sets where the snapshot and unsent events are kept. + * + * @param value a directory the agent may write; created if missing + * @return this builder + */ + public Builder cacheDirectory(Path value) + { + this.cacheDirectory = value; + return this; + } + + /** + * Sets the connection and answer timeouts. + * + * @param connect how long to wait for a connection + * @param read how long to wait for an answer + * @return this builder + */ + public Builder timeouts(Duration connect, Duration read) + { + this.connectTimeout = connect; + this.readTimeout = read; + return this; + } + + /** + * Sets the time between heartbeats until the server sets it. + * + * @param value the time + * @return this builder + */ + public Builder refreshInterval(Duration value) + { + this.refreshInterval = value; + return this; + } + + /** + * Sets how events are batched. + * + * @param batchSize the most events per batch, 1 to {@value #MAX_BATCH} + * @param flushInterval how often a partial batch is sent + * @param queueCapacity how many events wait in memory + * @return this builder + */ + public Builder audit(int batchSize, Duration flushInterval, int queueCapacity) + { + this.auditBatchSize = batchSize; + this.auditFlushInterval = flushInterval; + this.auditQueueCapacity = queueCapacity; + return this; + } + + /** + * Sets how much disk unsent events may take. + * + * @param value the size in bytes + * @return this builder + */ + public Builder spoolLimitBytes(long value) + { + this.spoolLimitBytes = value; + return this; + } + + /** + * Pins the server's public key instead of fetching it on first contact. + * + * @param value the key + * @return this builder + */ + public Builder trustedKey(SigningKey value) + { + this.trustedKey = value; + return this; + } + + /** + * Checks and builds the settings. + * + * @return the settings + * @throws IllegalArgumentException if a setting is missing or out of range + */ + public AgentSettings build() + { + URI address = server; + if (address == null || !("http".equals(address.getScheme()) || "https".equals(address.getScheme())) || address.getHost() == null) { + throw new IllegalArgumentException("the server must be an http or https address"); + } + String secret = token; + if (secret == null || blank(secret)) { + throw new IllegalArgumentException("the agent token is missing"); + } + String name = instance; + if (name == null || !INSTANCE.matcher(name).matches()) { + throw new IllegalArgumentException("the instance name must be 1-128 visible characters"); + } + Path directory = cacheDirectory; + if (directory == null) { + throw new IllegalArgumentException("the cache directory is missing"); + } + positive(connectTimeout, "connect timeout"); + positive(readTimeout, "read timeout"); + positive(refreshInterval, "refresh interval"); + positive(auditFlushInterval, "audit flush interval"); + if (auditBatchSize < 1 || auditBatchSize > MAX_BATCH) { + throw new IllegalArgumentException("the audit batch size must be 1 to " + MAX_BATCH); + } + if (auditQueueCapacity < auditBatchSize) { + throw new IllegalArgumentException("the audit queue must hold at least one batch"); + } + if (spoolLimitBytes < 0) { + throw new IllegalArgumentException("the spool limit must not be negative"); + } + return new AgentSettings(this, address, secret.trim(), name, directory); + } + + private static boolean blank(String value) + { + for (int index = 0; index < value.length(); index++) { + if (!Character.isWhitespace(value.charAt(index))) { + return false; + } + } + return true; + } + + private static void positive(Duration value, String what) + { + if (value.isNegative() || value.isZero()) { + throw new IllegalArgumentException("the " + what + " must be positive"); + } + } + } +} diff --git a/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AuditShipper.java b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AuditShipper.java new file mode 100644 index 00000000..86dd1b73 --- /dev/null +++ b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AuditShipper.java @@ -0,0 +1,219 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.jspecify.annotations.Nullable; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Ships access events to the server in batches. Recording never blocks the system: an event goes into a bounded queue, + * and when the queue is full it is dropped and counted. {@link #ship} sends what waits; a batch the server cannot take + * is written to the spool directory and sent again later, oldest first, while the spool stays under its limit. Events + * carry ids, so a batch sent twice is stored once. + */ +final class AuditShipper +{ + static final String SPOOL = "audit-spool"; + + private static final Logger LOG = Logger.getLogger(AuditShipper.class.getName()); + private static final ObjectMapper JSON = new ObjectMapper(); + private static final TypeReference>> BATCH = new TypeReference>>() + { + }; + + private final AgentSettings settings; + private final ServerClient client; + private final BlockingQueue queue; + private final Path spool; + private final AtomicLong dropped = new AtomicLong(); + private final AtomicLong spoolSequence = new AtomicLong(); + + AuditShipper(AgentSettings settings, ServerClient client) + { + this.settings = settings; + this.client = client; + this.queue = new ArrayBlockingQueue<>(settings.auditQueueCapacity()); + this.spool = settings.cacheDirectory().resolve(SPOOL); + } + + /** + * Queues an event. + * + * @param event the event + * @return {@code false} if the queue was full and the event was dropped + */ + boolean record(AccessEvent event) + { + if (queue.offer(event)) { + return true; + } + dropped.incrementAndGet(); + return false; + } + + /** + * Returns how many events were dropped because the queue was full or the spool over its limit. + * + * @return the count + */ + long dropped() + { + return dropped.get(); + } + + /** + * Returns how many events wait in memory. + * + * @return the count + */ + int waiting() + { + return queue.size(); + } + + /** + * Sends the events that wait, in batches; while the server takes them, also sends the spooled batches. A batch the + * server does not take is spooled, and shipping stops until the next call. + * + * @param everything {@code true} to send every waiting event, {@code false} to leave a last partial batch for later + * unless the flush interval has passed (the caller decides that by calling with {@code true}) + */ + // A list per batch is what batching is. + @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") + void ship(boolean everything) + { + boolean reachable = true; + while (queue.size() >= settings.auditBatchSize() || everything && !queue.isEmpty()) { + List events = new ArrayList<>(settings.auditBatchSize()); + queue.drainTo(events, settings.auditBatchSize()); + List> batch = new ArrayList<>(events.size()); + for (AccessEvent event : events) { + batch.add(event.fields()); + } + if (reachable) { + reachable = send(batch); + } + if (!reachable) { + spool(batch); + } + } + if (reachable) { + resend(); + } + } + + private boolean send(List> batch) + { + try { + client.send(batch); + return true; + } + catch (IOException unreachable) { + LOG.log(Level.FINE, "access events could not be sent; they are spooled", unreachable); + return false; + } + } + + /** Sends the spooled batches, oldest first, until one fails. */ + private void resend() + { + for (Path file : spooled()) { + List> batch; + try { + batch = JSON.readValue(file.toFile(), BATCH); + } + catch (IOException broken) { + LOG.log(Level.WARNING, "the spooled access events in " + file + " cannot be read; they are dropped", broken); + delete(file); + continue; + } + if (!send(batch)) { + return; + } + delete(file); + } + } + + private void spool(List> batch) + { + try { + Files.createDirectories(spool); + byte[] bytes = JSON.writeValueAsBytes(batch); + List files = spooled(); + long used = 0; + for (Path file : files) { + used += Files.size(file); + } + // The oldest batches make room; a batch larger than the whole spool is not kept. + for (Path file : files) { + if (used + bytes.length <= settings.spoolLimitBytes()) { + break; + } + used -= Files.size(file); + dropped.addAndGet(JSON.readValue(file.toFile(), BATCH).size()); + delete(file); + } + if (used + bytes.length > settings.spoolLimitBytes()) { + dropped.addAndGet(batch.size()); + return; + } + String name = String.format("%019d-%06d.json", System.currentTimeMillis(), spoolSequence.incrementAndGet() % 1_000_000); + Path temporary = spool.resolve(name + ".tmp"); + Files.write(temporary, bytes); + Files.move(temporary, spool.resolve(name)); + } + catch (IOException failed) { + dropped.addAndGet(batch.size()); + LOG.log(Level.WARNING, batch.size() + " access events could neither be sent nor spooled; they are lost", failed); + } + } + + /** The spooled batches, oldest first. */ + List spooled() + { + List files = new ArrayList<>(); + try (DirectoryStream entries = Files.newDirectoryStream(spool, "*.json")) { + for (Path entry : entries) { + files.add(entry); + } + } + catch (NoSuchFileException missing) { + return Collections.emptyList(); + } + catch (IOException unreadable) { + LOG.log(Level.WARNING, "the audit spool " + spool + " cannot be read", unreadable); + return Collections.emptyList(); + } + Collections.sort(files); + return files; + } + + private static void delete(Path file) + { + try { + Files.deleteIfExists(file); + } + catch (IOException stuck) { + LOG.log(Level.WARNING, "the spooled access events in " + file + " cannot be deleted", stuck); + } + } +} diff --git a/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/GrantForgeAgent.java b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/GrantForgeAgent.java new file mode 100644 index 00000000..8cec0377 --- /dev/null +++ b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/GrantForgeAgent.java @@ -0,0 +1,313 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import org.devlive.grantforge.policy.engine.AccessRequest; +import org.devlive.grantforge.policy.engine.ConditionEvaluator; +import org.jspecify.annotations.Nullable; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * An agent's link to GrantForge: keeps the policy snapshot of its service current and decides access with it, and + * ships the access events the system records. Thread-safe; {@link #decide} and {@link #record} never wait for the + * server. + * + *

On {@link #start} the agent takes the snapshot it stored last, if its signature still holds, then sends + * heartbeats as often as the server asks. When the server's policy version differs from the applied one, it + * downloads the snapshot, checks its signature with the server's key (pinned in the settings, or fetched once and + * kept), builds an engine from it and stores it; a snapshot that fails any step is not applied and the previous one + * stays. Until a snapshot is applied, every decision is {@link AgentDecision.Outcome#NOT_DETERMINED}: the agent of a + * system decides whether that falls back to the system's own checks or denies. + */ +// An agent keeps its policies current and ships events in the background, beside the system it lives in; the snapshot +// it applies is published to the system's threads through volatile fields. +@SuppressWarnings({"PMD.DoNotUseThreads", "PMD.AvoidUsingVolatile"}) +public final class GrantForgeAgent + implements AutoCloseable +{ + private static final Logger LOG = Logger.getLogger(GrantForgeAgent.class.getName()); + + private final AgentSettings settings; + private final Map evaluators; + private final ServerClient client; + private final SnapshotStore store; + private final AuditShipper shipper; + private final ScheduledExecutorService scheduler; + private volatile @Nullable Applied applied; + private volatile @Nullable SigningKey key; + private volatile boolean reachable = true; + + // The pool starts no thread until a task is scheduled, which only start() does. + GrantForgeAgent(AgentSettings settings, Map evaluators) + { + this.settings = settings; + this.evaluators = Collections.unmodifiableMap(new HashMap<>(evaluators)); + this.client = new ServerClient(settings); + this.store = new SnapshotStore(settings.cacheDirectory()); + this.shipper = new AuditShipper(settings, client); + this.key = settings.trustedKey(); + this.scheduler = Executors.newScheduledThreadPool(2, new AgentThreads(settings.instance())); + } + + /** + * Starts an agent: applies the stored snapshot, then keeps the snapshot current and ships events in the background. + * + * @param settings the settings + * @param evaluators the evaluators of the conditions the service type defines, by evaluator name + * @return the running agent; {@link #close} it when the system stops + */ + public static GrantForgeAgent start(AgentSettings settings, Map evaluators) + { + GrantForgeAgent agent = new GrantForgeAgent(settings, evaluators); + agent.loadStored(); + agent.scheduler.execute(agent::refreshAndReschedule); + long flush = settings.auditFlushInterval().toMillis(); + agent.scheduler.scheduleWithFixedDelay(agent::shipQuietly, flush, flush, TimeUnit.MILLISECONDS); + return agent; + } + + /** + * Decides a request with the applied snapshot. + * + * @param request the request; the agent adds the roles and groups the snapshot gives its user + * @return the decision; {@link AgentDecision.Outcome#NOT_DETERMINED} without a snapshot + */ + public AgentDecision decide(AccessRequest request) + { + Applied current = applied; + return current == null ? AgentDecision.withoutSnapshot() : current.snapshot.decide(request); + } + + /** + * Queues an access event for the server. + * + * @param event the event + * @return {@code false} if too many events wait and this one was dropped + */ + public boolean record(AccessEvent event) + { + return shipper.record(event); + } + + /** + * Returns the applied snapshot. + * + * @return the snapshot, or {@code null} before one is applied + */ + public @Nullable Snapshot snapshot() + { + Applied current = applied; + return current == null ? null : current.snapshot; + } + + /** + * Returns whether the last call to the server succeeded. + * + * @return {@code true} if it did + */ + public boolean serverReachable() + { + return reachable; + } + + /** + * Returns how many access events were dropped, because too many waited or the spool was full. + * + * @return the count + */ + public long droppedEvents() + { + return shipper.dropped(); + } + + /** Applies the stored snapshot, if there is one whose signature holds. */ + void loadStored() + { + try { + SnapshotStore.Stored stored = store.load(settings.trustedKey()); + if (stored == null) { + return; + } + apply(stored.snapshot(), stored.key(), false); + LOG.info("Applied the stored policy snapshot of version " + policyVersion()); + } + catch (IOException | IllegalArgumentException unusable) { + LOG.log(Level.WARNING, "The stored policy snapshot cannot be used; waiting for the server", unusable); + } + } + + /** + * Sends a heartbeat and applies a new snapshot if the policy version changed. + * + * @return the seconds until the next heartbeat the server asks for, or the configured interval if it cannot be + * reached + */ + long refresh() + { + long fallback = settings.refreshInterval().getSeconds(); + try { + Long version = policyVersion(); + ServerClient.Heartbeat heartbeat = client.heartbeat(version); + if (version == null || heartbeat.policyVersion() != version) { + Applied current = applied; + ServerClient.Download download = client.policies(current == null ? null : current.etag); + if (download.changed()) { + apply(download, keyFor(download.keyId()), true); + LOG.info("Applied policy snapshot version " + policyVersion()); + } + } + reachable = true; + return heartbeat.refreshSeconds() > 0 ? heartbeat.refreshSeconds() : fallback; + } + catch (IOException unreachable) { + if (reachable) { + LOG.log(Level.WARNING, "GrantForge cannot be reached; keeping the applied policies", unreachable); + } + reachable = false; + return fallback; + } + catch (IllegalArgumentException | SecurityException rejected) { + reachable = true; + LOG.log(Level.WARNING, "The policy snapshot was rejected; keeping the applied policies", rejected); + return fallback; + } + } + + /** The key that signed a snapshot: the pinned one, the one fetched before, or the server's current one. */ + private SigningKey keyFor(String keyId) throws IOException + { + SigningKey known = key; + if (known != null && known.keyId().equals(keyId)) { + return known; + } + if (settings.trustedKey() != null) { + throw new SecurityException("the snapshot is signed with key " + keyId + ", not with the pinned key"); + } + SigningKey fetched = client.signingKey(); + if (!fetched.keyId().equals(keyId)) { + throw new SecurityException("the snapshot is signed with key " + keyId + ", the server publishes " + fetched.keyId()); + } + key = fetched; + return fetched; + } + + private void apply(ServerClient.Download download, SigningKey signer, boolean save) throws IOException + { + if (!signer.keyId().equals(download.keyId()) || !signer.verifies(download.body(), download.signature())) { + throw new SecurityException("the policy snapshot's signature does not hold"); + } + Snapshot snapshot = Snapshot.parse(download.body(), evaluators); + applied = new Applied(snapshot, download.etag()); + key = signer; + if (save) { + try { + store.save(download, signer); + } + catch (IOException unwritable) { + LOG.log(Level.WARNING, "The policy snapshot cannot be stored in " + settings.cacheDirectory(), unwritable); + } + } + } + + private @Nullable Long policyVersion() + { + Applied current = applied; + return current == null ? null : current.snapshot.policyVersion(); + } + + /** + * Sends the waiting events. + */ + void ship() + { + shipper.ship(true); + } + + // A failure here must not stop the scheduled task from running again. + private void shipQuietly() + { + try { + ship(); + } + catch (RuntimeException unexpected) { + LOG.log(Level.WARNING, "Shipping access events failed", unexpected); + } + } + + // A failure here must not end the heartbeats. + private void refreshAndReschedule() + { + long delay = settings.refreshInterval().getSeconds(); + try { + delay = refresh(); + } + catch (RuntimeException unexpected) { + LOG.log(Level.WARNING, "Refreshing the policies failed", unexpected); + } + if (!scheduler.isShutdown()) { + scheduler.schedule(this::refreshAndReschedule, delay, TimeUnit.SECONDS); + } + } + + /** Stops the background work and sends the events that wait, or spools them. */ + @Override + public void close() + { + scheduler.shutdownNow(); + try { + scheduler.awaitTermination(settings.readTimeout().toMillis(), TimeUnit.MILLISECONDS); + } + catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + ship(); + } + + /** The applied snapshot with the ETag it was downloaded with. */ + private static final class Applied + { + final Snapshot snapshot; + final String etag; + + Applied(Snapshot snapshot, String etag) + { + this.snapshot = snapshot; + this.etag = etag; + } + } + + /** Daemon threads named after the agent, so they never keep the system from stopping. */ + private static final class AgentThreads + implements ThreadFactory + { + private final String instance; + private final AtomicInteger count = new AtomicInteger(); + + AgentThreads(String instance) + { + this.instance = instance; + } + + @Override + public Thread newThread(Runnable task) + { + Thread thread = new Thread(task, "grantforge-agent-" + instance + "-" + count.incrementAndGet()); + thread.setDaemon(true); + return thread; + } + } +} diff --git a/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/ServerClient.java b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/ServerClient.java new file mode 100644 index 00000000..e874041d --- /dev/null +++ b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/ServerClient.java @@ -0,0 +1,289 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.jspecify.annotations.Nullable; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Calls the agent API of the server ({@code /api/v1/agent/**}) with the service's token. Thread-safe; every call opens + * its own connection, which {@link HttpURLConnection} keeps alive between calls. + */ +final class ServerClient +{ + static final String SIGNATURE_HEADER = "X-GrantForge-Signature"; + static final String KEY_HEADER = "X-GrantForge-Signing-Key"; + static final String VERSION_HEADER = "X-GrantForge-Policy-Version"; + + /** Largest answer read; a snapshot of a large service is a few megabytes. */ + static final int MAX_BODY = 64 * 1024 * 1024; + + private static final ObjectMapper JSON = new ObjectMapper(); + + private final AgentSettings settings; + private final URI base; + + ServerClient(AgentSettings settings) + { + this.settings = settings; + String address = settings.server().toString(); + this.base = URI.create(address.endsWith("/") ? address : address + "/"); + } + + /** + * Reports that the agent is alive and learns the current policy version. + * + * @param appliedPolicyVersion the version the agent applies, or {@code null} before its first snapshot + * @return the server's answer + * @throws IOException if the server cannot be reached or refuses + */ + Heartbeat heartbeat(@Nullable Long appliedPolicyVersion) throws IOException + { + Map body = new LinkedHashMap<>(); + body.put("instance", settings.instance()); + body.put("host", settings.host()); + body.put("agentVersion", settings.agentVersion()); + body.put("appliedPolicyVersion", appliedPolicyVersion); + JsonNode answer = json(send("POST", "api/v1/agent/heartbeat", JSON.writeValueAsBytes(body), null).body); + return new Heartbeat(answer.path("policyVersion").asLong(), answer.path("refreshSeconds").asLong()); + } + + /** + * Downloads the service's policy snapshot, unless the agent has it already. + * + * @param etag the ETag of the snapshot the agent has, or {@code null} + * @return the snapshot, or that it has not changed + * @throws IOException if the server cannot be reached or refuses + */ + Download policies(@Nullable String etag) throws IOException + { + Response response = send("GET", "api/v1/agent/policies", null, etag); + if (response.status == HttpURLConnection.HTTP_NOT_MODIFIED) { + return Download.unchanged(); + } + String signature = response.header(SIGNATURE_HEADER); + String keyId = response.header(KEY_HEADER); + String tag = response.header("ETag"); + if (signature == null || keyId == null || tag == null) { + throw new IOException("the snapshot came without its ETag, signature or key id"); + } + return new Download(response.body, tag, keyId, signature); + } + + /** + * Fetches the public key snapshots are signed with. + * + * @return the key + * @throws IOException if the server cannot be reached or refuses + * @throws IllegalArgumentException if the answer is not an Ed25519 key + */ + SigningKey signingKey() throws IOException + { + JsonNode answer = json(send("GET", "api/v1/agent/signing-key", null, null).body); + return SigningKey.of(answer.path("publicKey").asText("")); + } + + /** + * Sends a batch of access events. + * + * @param events the events, as {@link AccessEvent#fields()} gives them + * @return how many the server stored + * @throws IOException if the server cannot be reached or refuses + */ + int send(List> events) throws IOException + { + Map body = new LinkedHashMap<>(); + body.put("instance", settings.instance()); + body.put("events", events); + JsonNode answer = json(send("POST", "api/v1/agent/access-events", JSON.writeValueAsBytes(body), null).body); + return answer.path("accepted").asInt() + answer.path("duplicates").asInt(); + } + + private Response send(String method, String path, byte @Nullable [] body, @Nullable String etag) throws IOException + { + URL url = base.resolve(path).toURL(); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + try { + connection.setRequestMethod(method); + connection.setConnectTimeout((int) settings.connectTimeout().toMillis()); + connection.setReadTimeout((int) settings.readTimeout().toMillis()); + connection.setInstanceFollowRedirects(false); + connection.setUseCaches(false); + connection.setRequestProperty("Authorization", "Bearer " + settings.token()); + connection.setRequestProperty("Accept", "application/json"); + connection.setRequestProperty("User-Agent", "grantforge-agent/" + settings.agentVersion()); + if (etag != null) { + connection.setRequestProperty("If-None-Match", etag); + } + if (body != null) { + connection.setDoOutput(true); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setFixedLengthStreamingMode(body.length); + try (OutputStream out = connection.getOutputStream()) { + out.write(body); + } + } + int status = connection.getResponseCode(); + if (status == HttpURLConnection.HTTP_NOT_MODIFIED) { + return new Response(status, new byte[0], connection); + } + if (status / 100 != 2) { + throw new IOException(method + " " + path + " answered " + status + ": " + text(read(connection.getErrorStream()))); + } + return new Response(status, read(connection.getInputStream()), connection); + } + finally { + connection.disconnect(); + } + } + + private static byte[] read(@Nullable InputStream stream) throws IOException + { + if (stream == null) { + return new byte[0]; + } + try (InputStream in = stream) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read = in.read(buffer); + while (read != -1) { + if (out.size() + read > MAX_BODY) { + throw new IOException("the answer is larger than " + MAX_BODY + " bytes"); + } + out.write(buffer, 0, read); + read = in.read(buffer); + } + return out.toByteArray(); + } + } + + private static String text(byte[] body) + { + String text = new String(body, StandardCharsets.UTF_8); + return text.length() > 300 ? text.substring(0, 300) + "..." : text; + } + + private static JsonNode json(byte[] body) throws IOException + { + JsonNode node = JSON.readTree(body); + if (node == null || !node.isObject()) { + throw new IOException("the server's answer is not a JSON object"); + } + return node; + } + + /** An answer, read before the connection is let go; it owns the body it was read into. */ + @SuppressWarnings("PMD.ArrayIsStoredDirectly") + private static final class Response + { + final int status; + final byte[] body; + private final Map headers = new LinkedHashMap<>(); + + Response(int status, byte[] body, HttpURLConnection connection) + { + this.status = status; + this.body = body; + for (String name : new String[] {SIGNATURE_HEADER, KEY_HEADER, VERSION_HEADER, "ETag"}) { + String value = connection.getHeaderField(name); + if (value != null) { + headers.put(name, value); + } + } + } + + @Nullable String header(String name) + { + return headers.get(name); + } + } + + /** The answer to a heartbeat. */ + static final class Heartbeat + { + private final long policyVersion; + private final long refreshSeconds; + + Heartbeat(long policyVersion, long refreshSeconds) + { + this.policyVersion = policyVersion; + this.refreshSeconds = refreshSeconds; + } + + long policyVersion() + { + return policyVersion; + } + + long refreshSeconds() + { + return refreshSeconds; + } + } + + /** A downloaded snapshot, or that the agent's is current. Holds the body as it is: snapshots run to megabytes. */ + @SuppressWarnings({"PMD.ArrayIsStoredDirectly", "PMD.MethodReturnsInternalArray"}) + static final class Download + { + private static final Download UNCHANGED = new Download(new byte[0], "", "", ""); + + private final byte[] body; + private final String etag; + private final String keyId; + private final String signature; + + Download(byte[] body, String etag, String keyId, String signature) + { + this.body = body; + this.etag = etag; + this.keyId = keyId; + this.signature = signature; + } + + static Download unchanged() + { + return UNCHANGED; + } + + boolean changed() + { + return this != UNCHANGED; + } + + byte[] body() + { + return body; + } + + String etag() + { + return etag; + } + + String keyId() + { + return keyId; + } + + String signature() + { + return signature; + } + } +} diff --git a/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/SigningKey.java b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/SigningKey.java new file mode 100644 index 00000000..0e466c48 --- /dev/null +++ b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/SigningKey.java @@ -0,0 +1,122 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import org.bouncycastle.crypto.params.AsymmetricKeyParameter; +import org.bouncycastle.crypto.params.Ed25519PublicKeyParameters; +import org.bouncycastle.crypto.signers.Ed25519Signer; +import org.bouncycastle.crypto.util.PublicKeyFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; + +/** + * The server's Ed25519 public key that policy snapshots are signed with, as {@code GET /api/v1/agent/signing-key} + * returns it: X.509-encoded and Base64. Immutable. + */ +public final class SigningKey +{ + private final String encoded; + private final String keyId; + private final Ed25519PublicKeyParameters key; + + private SigningKey(String encoded, String keyId, Ed25519PublicKeyParameters key) + { + this.encoded = encoded; + this.keyId = keyId; + this.key = key; + } + + /** + * Reads a public key. + * + * @param encoded the key, X.509 SubjectPublicKeyInfo, Base64 + * @return the key + * @throws IllegalArgumentException if it is not an Ed25519 public key + */ + public static SigningKey of(String encoded) + { + String trimmed = encoded.trim(); + byte[] bytes; + AsymmetricKeyParameter parameters; + try { + bytes = Base64.getDecoder().decode(trimmed); + parameters = PublicKeyFactory.createKey(bytes); + } + catch (IOException | IllegalArgumentException | IllegalStateException broken) { + throw new IllegalArgumentException("not an X.509 public key", broken); + } + if (!(parameters instanceof Ed25519PublicKeyParameters)) { + throw new IllegalArgumentException("not an Ed25519 public key"); + } + return new SigningKey(trimmed, keyId(bytes), (Ed25519PublicKeyParameters) parameters); + } + + /** + * The id the server gives a key, which snapshots name in their {@code X-GrantForge-Signing-Key} header: the first + * 16 hexadecimal digits of the SHA-256 of its X.509 form. + */ + private static String keyId(byte[] encoded) + { + byte[] digest; + try { + digest = MessageDigest.getInstance("SHA-256").digest(encoded); + } + catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 is not available", impossible); + } + StringBuilder hex = new StringBuilder(); + for (int index = 0; index < 8; index++) { + hex.append(String.format("%02x", digest[index] & 0xff)); + } + return hex.toString(); + } + + /** + * Returns the key as it was given. + * + * @return the key, X.509, Base64 + */ + public String encoded() + { + return encoded; + } + + /** + * Returns the key's id. + * + * @return the id + */ + public String keyId() + { + return keyId; + } + + /** + * Checks a signature. + * + * @param data what was signed + * @param signature the signature, Base64 + * @return {@code true} if this key made the signature over the data + */ + public boolean verifies(byte[] data, String signature) + { + byte[] bytes; + try { + bytes = Base64.getDecoder().decode(signature.trim().getBytes(StandardCharsets.US_ASCII)); + } + catch (IllegalArgumentException broken) { + return false; + } + Ed25519Signer verifier = new Ed25519Signer(); + verifier.init(false, key); + verifier.update(data, 0, data.length); + return verifier.verifySignature(bytes); + } +} diff --git a/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/Snapshot.java b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/Snapshot.java new file mode 100644 index 00000000..ac5152fd --- /dev/null +++ b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/Snapshot.java @@ -0,0 +1,484 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.devlive.grantforge.policy.engine.AccessRequest; +import org.devlive.grantforge.policy.engine.Condition; +import org.devlive.grantforge.policy.engine.ConditionEvaluator; +import org.devlive.grantforge.policy.engine.Decision; +import org.devlive.grantforge.policy.engine.MatcherKind; +import org.devlive.grantforge.policy.engine.Policy; +import org.devlive.grantforge.policy.engine.PolicyEngine; +import org.devlive.grantforge.policy.engine.PolicyItem; +import org.devlive.grantforge.policy.engine.Priority; +import org.devlive.grantforge.policy.engine.ResourceLevel; +import org.devlive.grantforge.policy.engine.ResourceSpec; +import org.devlive.grantforge.policy.engine.ServiceModel; +import org.devlive.grantforge.policy.engine.Validity; +import org.jspecify.annotations.Nullable; + +import java.io.IOException; +import java.time.Instant; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * A policy snapshot of the agent's service, read from the JSON the server signs, with an engine for its access policies + * and the roles and groups the snapshot gives each user. Immutable and thread-safe. + * + *

Masking and row filtering policies are counted, not evaluated: the agents of systems that apply them read them + * themselves. + */ +public final class Snapshot +{ + /** The snapshot format this agent reads; the server raises it on incompatible changes. */ + public static final int FORMAT = 1; + + private static final ObjectMapper JSON = new ObjectMapper(); + + private final String service; + private final String serviceType; + private final boolean serviceEnabled; + private final long policyVersion; + private final PolicyEngine engine; + private final int accessPolicies; + private final int otherPolicies; + private final Map> rolesOfUser; + private final Map> groupsOfUser; + + private Snapshot(String service, String serviceType, boolean serviceEnabled, long policyVersion, PolicyEngine engine, + int accessPolicies, int otherPolicies, Map> rolesOfUser, Map> groupsOfUser) + { + this.service = service; + this.serviceType = serviceType; + this.serviceEnabled = serviceEnabled; + this.policyVersion = policyVersion; + this.engine = engine; + this.accessPolicies = accessPolicies; + this.otherPolicies = otherPolicies; + this.rolesOfUser = rolesOfUser; + this.groupsOfUser = groupsOfUser; + } + + /** + * Reads a snapshot. + * + * @param body the snapshot as the server sends it + * @param evaluators the condition evaluators the agent has, by evaluator name; conditions whose evaluator is missing + * are taken in the safe direction (see {@link PolicyEngine}) + * @return the snapshot + * @throws IllegalArgumentException if it is not a snapshot of a format this agent reads, or its policies do not fit + * its service type + */ + public static Snapshot parse(byte[] body, Map evaluators) + { + JsonNode root; + try { + root = JSON.readTree(body); + } + catch (IOException broken) { + throw new IllegalArgumentException("the snapshot is not valid JSON", broken); + } + if (root == null || !root.isObject()) { + throw new IllegalArgumentException("the snapshot is not a JSON object"); + } + long format = number(root, "format"); + if (format != FORMAT) { + throw new IllegalArgumentException("snapshot format " + format + " is not supported; this agent reads format " + FORMAT + + ", upgrade it"); + } + JsonNode definition = object(root, "definition"); + ServiceModel model = model(definition); + Map conditions = conditions(definition, evaluators); + List policies = new ArrayList<>(); + int others = 0; + for (JsonNode policy : array(root, "policies")) { + if ("ACCESS".equals(text(policy, "type"))) { + policies.add(policy(policy)); + } + else { + others++; + } + } + return new Snapshot(text(root, "service"), text(root, "serviceType"), bool(root, "serviceEnabled"), number(root, "policyVersion"), + PolicyEngine.create(model, policies, conditions), policies.size(), others, byUser(root, "roles"), byUser(root, "groups")); + } + + private static ServiceModel model(JsonNode definition) + { + // The engine wants every level after its parent; the definition lists them in the service type's order. + Map pending = new LinkedHashMap<>(); + for (JsonNode level : array(definition, "resources")) { + pending.put(text(level, "name"), level); + } + ServiceModel.Builder model = ServiceModel.builder(); + Set added = new LinkedHashSet<>(); + while (!pending.isEmpty()) { + boolean progress = false; + Iterator> levels = pending.entrySet().iterator(); + while (levels.hasNext()) { + JsonNode level = levels.next().getValue(); + String parent = optionalText(level, "parent"); + if (parent == null || added.contains(parent)) { + String name = text(level, "name"); + model.level(ResourceLevel.of(name, parent, matcher(text(level, "matcher")), bool(level, "caseSensitive"))); + added.add(name); + levels.remove(); + progress = true; + } + } + if (!progress) { + throw new IllegalArgumentException("resource levels " + pending.keySet() + " have unknown parents"); + } + } + for (JsonNode access : array(definition, "accessTypes")) { + List implied = texts(access, "impliedGrants"); + model.accessType(text(access, "name"), implied.toArray(new String[0])); + } + return model.build(); + } + + private static MatcherKind matcher(String name) + { + try { + return MatcherKind.valueOf(name); + } + catch (IllegalArgumentException unknown) { + throw new IllegalArgumentException("matcher " + name + " is not one this agent knows; upgrade it", unknown); + } + } + + private static Map conditions(JsonNode definition, Map evaluators) + { + Map conditions = new HashMap<>(); + for (JsonNode condition : array(definition, "conditions")) { + ConditionEvaluator evaluator = evaluators.get(text(condition, "evaluator")); + if (evaluator != null) { + conditions.put(text(condition, "name"), evaluator); + } + } + return conditions; + } + + private static Policy policy(JsonNode node) + { + String id = text(node, "id"); + long policyId; + try { + policyId = Long.parseLong(id); + } + catch (NumberFormatException broken) { + throw new IllegalArgumentException("policy id " + id + " is not a number", broken); + } + JsonNode document = object(node, "document"); + Policy.Builder policy = Policy.builder(policyId).priority(Priority.valueOf(text(node, "priority"))); + Iterator> resources = object(document, "resources").fields(); + while (resources.hasNext()) { + Map.Entry resource = resources.next(); + JsonNode values = resource.getValue(); + policy.resource(resource.getKey(), ResourceSpec.of(texts(values, "values"), bool(values, "excludes"), bool(values, "recursive"))); + } + policy.allow(items(document, "allow")); + policy.allowExceptions(items(document, "allowExceptions")); + policy.deny(items(document, "deny")); + policy.denyExceptions(items(document, "denyExceptions")); + for (JsonNode period : optionalArray(document, "validity")) { + policy.validity(Validity.between(instant(period, "from"), instant(period, "until"))); + } + return policy.build(); + } + + // One item per item of the document. + @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") + private static PolicyItem[] items(JsonNode document, String name) + { + List items = new ArrayList<>(); + for (JsonNode item : optionalArray(document, name)) { + List conditions = new ArrayList<>(); + for (JsonNode condition : optionalArray(item, "conditions")) { + conditions.add(Condition.of(text(condition, "type"), texts(condition, "values").toArray(new String[0]))); + } + items.add(PolicyItem.builder() + .users(texts(item, "users").toArray(new String[0])) + .groups(texts(item, "groups").toArray(new String[0])) + .roles(texts(item, "roles").toArray(new String[0])) + .accessTypes(texts(item, "accessTypes").toArray(new String[0])) + .conditions(conditions.toArray(new Condition[0])) + .build()); + } + return items.toArray(new PolicyItem[0]); + } + + /** Turns "role: its users" around into "user: their roles". */ + // One set per user. + @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") + private static Map> byUser(JsonNode root, String name) + { + JsonNode holders = root.get(name); + if (holders == null || holders.isNull()) { + return Collections.emptyMap(); + } + if (!holders.isObject()) { + throw new IllegalArgumentException(name + " is not an object"); + } + Map> byUser = new HashMap<>(); + Iterator> entries = holders.fields(); + while (entries.hasNext()) { + Map.Entry entry = entries.next(); + if (!entry.getValue().isArray()) { + throw new IllegalArgumentException("the members of " + entry.getKey() + " are not an array"); + } + for (JsonNode user : entry.getValue()) { + if (!user.isTextual()) { + throw new IllegalArgumentException("a member of " + entry.getKey() + " is not a name"); + } + Set of = byUser.get(user.asText()); + if (of == null) { + of = new LinkedHashSet<>(); + byUser.put(user.asText(), of); + } + of.add(entry.getKey()); + } + } + for (Map.Entry> entry : byUser.entrySet()) { + entry.setValue(Collections.unmodifiableSet(entry.getValue())); + } + return Collections.unmodifiableMap(byUser); + } + + /** + * Decides a request: adds the roles and groups the snapshot gives the user to those the system gave, then asks the + * engine. A service that is not in use decides nothing, so the system's own checks apply. + * + * @param request the request + * @return the decision + */ + public AgentDecision decide(AccessRequest request) + { + if (!serviceEnabled) { + return AgentDecision.notDetermined(policyVersion); + } + Decision decision = engine.evaluate(enrich(request)); + Long policy = decision.policyId(); + switch (decision.outcome()) { + case ALLOWED: + return AgentDecision.allowed(policyVersion, policy == null ? 0 : policy); + case DENIED: + return AgentDecision.denied(policyVersion, policy == null ? 0 : policy); + default: + return AgentDecision.notDetermined(policyVersion); + } + } + + /** + * Adds the roles and groups the snapshot gives the request's user. + * + * @param request the request + * @return the request with them + */ + AccessRequest enrich(AccessRequest request) + { + Set groups = new LinkedHashSet<>(request.groups()); + groups.addAll(groupsOf(request.user())); + Set roles = new LinkedHashSet<>(request.roles()); + roles.addAll(rolesOf(request.user())); + AccessRequest.Builder copy = AccessRequest.builder(request.user(), request.accessType()) + .groups(groups.toArray(new String[0])) + .roles(roles.toArray(new String[0])) + .time(request.time()) + .context(request.context()); + for (Map.Entry level : request.resource().entrySet()) { + copy.resource(level.getKey(), level.getValue()); + } + return copy.build(); + } + + /** + * Returns the roles the snapshot gives a user. + * + * @param user the user name + * @return the role codes + */ + public Set rolesOf(String user) + { + Set roles = rolesOfUser.get(user); + return roles == null ? Collections.emptySet() : roles; + } + + /** + * Returns the groups the snapshot puts a user in. + * + * @param user the user name + * @return the group codes + */ + public Set groupsOf(String user) + { + Set groups = groupsOfUser.get(user); + return groups == null ? Collections.emptySet() : groups; + } + + /** + * Returns the service's name. + * + * @return the name + */ + public String service() + { + return service; + } + + /** + * Returns the service type's name. + * + * @return the name + */ + public String serviceType() + { + return serviceType; + } + + /** + * Returns whether the service is in use; a service that is not decides nothing. + * + * @return {@code true} if it is + */ + public boolean serviceEnabled() + { + return serviceEnabled; + } + + /** + * Returns the policy version of the snapshot. + * + * @return the version + */ + public long policyVersion() + { + return policyVersion; + } + + /** + * Returns how many access policies the engine holds. + * + * @return the count + */ + public int accessPolicies() + { + return accessPolicies; + } + + /** + * Returns how many masking and row filtering policies the snapshot holds. + * + * @return the count + */ + public int otherPolicies() + { + return otherPolicies; + } + + private static JsonNode member(JsonNode node, String name) + { + JsonNode value = node.get(name); + if (value == null || value.isNull()) { + throw new IllegalArgumentException(name + " is missing"); + } + return value; + } + + private static String text(JsonNode node, String name) + { + JsonNode value = member(node, name); + if (!value.isTextual()) { + throw new IllegalArgumentException(name + " is not a string"); + } + return value.asText(); + } + + private static @Nullable String optionalText(JsonNode node, String name) + { + JsonNode value = node.get(name); + return value == null || value.isNull() ? null : text(node, name); + } + + private static long number(JsonNode node, String name) + { + JsonNode value = member(node, name); + if (!value.isIntegralNumber() || !value.canConvertToLong()) { + throw new IllegalArgumentException(name + " is not a whole number"); + } + return value.asLong(); + } + + private static boolean bool(JsonNode node, String name) + { + JsonNode value = member(node, name); + if (!value.isBoolean()) { + throw new IllegalArgumentException(name + " is not a boolean"); + } + return value.asBoolean(); + } + + private static JsonNode object(JsonNode node, String name) + { + JsonNode value = member(node, name); + if (!value.isObject()) { + throw new IllegalArgumentException(name + " is not an object"); + } + return value; + } + + private static JsonNode array(JsonNode node, String name) + { + JsonNode value = member(node, name); + if (!value.isArray()) { + throw new IllegalArgumentException(name + " is not an array"); + } + return value; + } + + private static Iterable optionalArray(JsonNode node, String name) + { + JsonNode value = node.get(name); + return value == null || value.isNull() ? Collections.emptyList() : array(node, name); + } + + private static List texts(JsonNode node, String name) + { + List texts = new ArrayList<>(); + for (JsonNode value : optionalArray(node, name)) { + if (!value.isTextual()) { + throw new IllegalArgumentException(name + " holds something other than strings"); + } + texts.add(value.asText()); + } + return texts; + } + + private static @Nullable Instant instant(JsonNode node, String name) + { + String value = optionalText(node, name); + if (value == null) { + return null; + } + try { + return Instant.parse(value); + } + catch (DateTimeParseException broken) { + throw new IllegalArgumentException(name + " is not an instant", broken); + } + } +} diff --git a/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/SnapshotStore.java b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/SnapshotStore.java new file mode 100644 index 00000000..7998e17a --- /dev/null +++ b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/SnapshotStore.java @@ -0,0 +1,133 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import org.jspecify.annotations.Nullable; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Properties; + +/** + * Keeps the last good snapshot on disk with its signature and the key that made it, so an agent that starts while the + * server cannot be reached still decides with the policies it had. The signature is checked again when the snapshot + * is read back, so a file changed on disk is not used. + */ +final class SnapshotStore +{ + static final String BODY = "snapshot.json"; + static final String META = "snapshot.properties"; + + private final Path directory; + + SnapshotStore(Path directory) + { + this.directory = directory; + } + + /** + * Stores a snapshot, replacing the one stored. + * + * @param snapshot the snapshot as downloaded + * @param key the key whose signature was checked + * @throws IOException if it cannot be written + */ + void save(ServerClient.Download snapshot, SigningKey key) throws IOException + { + Files.createDirectories(directory); + Properties meta = new Properties(); + meta.setProperty("etag", snapshot.etag()); + meta.setProperty("keyId", snapshot.keyId()); + meta.setProperty("signature", snapshot.signature()); + meta.setProperty("signingKey", key.encoded()); + Path body = Files.write(directory.resolve(BODY + ".tmp"), snapshot.body()); + Path properties = directory.resolve(META + ".tmp"); + try (OutputStream out = Files.newOutputStream(properties)) { + meta.store(out, "The policy snapshot in " + BODY); + } + move(body, directory.resolve(BODY)); + move(properties, directory.resolve(META)); + } + + /** + * Reads the stored snapshot back if its signature still holds. + * + * @param trusted the key the agent is pinned to, or {@code null} to accept the stored key + * @return the snapshot and its key, or {@code null} if none is stored, or it was changed or signed by another key + * @throws IOException if the files exist but cannot be read + */ + @Nullable Stored load(@Nullable SigningKey trusted) throws IOException + { + byte[] body; + Properties meta = new Properties(); + try { + body = Files.readAllBytes(directory.resolve(BODY)); + try (InputStream in = Files.newInputStream(directory.resolve(META))) { + meta.load(in); + } + } + catch (NoSuchFileException missing) { + return null; + } + String etag = meta.getProperty("etag"); + String keyId = meta.getProperty("keyId"); + String signature = meta.getProperty("signature"); + String encoded = meta.getProperty("signingKey"); + if (etag == null || keyId == null || signature == null || encoded == null) { + return null; + } + SigningKey key; + try { + key = SigningKey.of(encoded); + } + catch (IllegalArgumentException broken) { + return null; + } + if (trusted != null && !trusted.keyId().equals(key.keyId()) || !key.keyId().equals(keyId) || !key.verifies(body, signature)) { + return null; + } + return new Stored(new ServerClient.Download(body, etag, keyId, signature), key); + } + + private static void move(Path from, Path to) throws IOException + { + try { + Files.move(from, to, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } + catch (AtomicMoveNotSupportedException unsupported) { + Files.move(from, to, StandardCopyOption.REPLACE_EXISTING); + } + } + + /** A stored snapshot with the key that signed it. */ + static final class Stored + { + private final ServerClient.Download snapshot; + private final SigningKey key; + + Stored(ServerClient.Download snapshot, SigningKey key) + { + this.snapshot = snapshot; + this.key = key; + } + + ServerClient.Download snapshot() + { + return snapshot; + } + + SigningKey key() + { + return key; + } + } +} diff --git a/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/package-info.java b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/package-info.java new file mode 100644 index 00000000..0b70950d --- /dev/null +++ b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/package-info.java @@ -0,0 +1,21 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +/** + * The part every agent shares, whatever system it protects (M13-01, D-84). A + * {@link org.devlive.grantforge.agent.GrantForgeAgent} keeps the policy snapshot of its service current: it sends + * heartbeats, downloads a new snapshot when the policy version changes, checks the snapshot's Ed25519 signature and + * keeps the last good one on disk for when the server cannot be reached. It decides access with the policy engine, + * adding the roles and groups the snapshot gives the user, and ships access events in batches, spooling them to disk + * while the server is away. + * + *

Java 8 on {@link java.net.HttpURLConnection}, with Jackson 2 for JSON and Bouncy Castle for Ed25519, which the JDK + * has only from Java 15 on. Agents run inside the protected systems, whose class paths bring their own versions of + * such libraries, so an agent relocates them when it packages itself. + */ +@NullMarked +package org.devlive.grantforge.agent; + +import org.jspecify.annotations.NullMarked; diff --git a/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AccessEventTest.java b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AccessEventTest.java new file mode 100644 index 00000000..dfc687e2 --- /dev/null +++ b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AccessEventTest.java @@ -0,0 +1,47 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class AccessEventTest +{ + @Test + void writesTheFieldsTheServerTakes() + { + AccessEvent event = AccessEvent.builder("alice", "sales.orders.id", "select", true).decidedBy(AgentDecision.allowed(4, 11)) + .occurredAt(Instant.parse("2026-10-05T01:02:03Z")).clientIp("10.1.2.3").resourceType("column").action("QUERY") + .request("SELECT id FROM orders").build(); + + Map fields = event.fields(); + assertThat(fields).containsEntry("eventId", event.eventId()).containsEntry("occurredAt", "2026-10-05T01:02:03Z") + .containsEntry("user", "alice").containsEntry("clientIp", "10.1.2.3").containsEntry("resource", "sales.orders.id") + .containsEntry("resourceType", "column").containsEntry("accessType", "select").containsEntry("action", "QUERY") + .containsEntry("outcome", "ALLOWED").containsEntry("policyId", "11").containsEntry("policyVersion", 4L) + .containsEntry("enforcer", "GRANTFORGE").containsEntry("request", "SELECT id FROM orders"); + assertThat(event.eventId()).hasSize(36); + } + + @Test + void creditsTheSystemWhenGrantForgeDidNotDecide() + { + Map fields = AccessEvent.builder("bob", "/data", "read", false).decidedBy(AgentDecision.notDetermined(4)) + .request("x".repeat(1500)).build().fields(); + + assertThat(fields).containsEntry("outcome", "DENIED").containsEntry("enforcer", "NATIVE").containsEntry("policyId", null) + .containsEntry("policyVersion", 4L).containsEntry("clientIp", null); + assertThat((String) fields.get("request")).hasSize(AccessEvent.MAX_REQUEST); + assertThat(AccessEvent.builder("bob", "/data", "read", true).build().fields()).containsEntry("enforcer", "NATIVE") + .containsEntry("request", null); + assertThat(AccessEvent.builder("a", "r", "t", true).build().eventId()) + .isNotEqualTo(AccessEvent.builder("a", "r", "t", true).build().eventId()); + } +} diff --git a/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AgentDecisionTest.java b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AgentDecisionTest.java new file mode 100644 index 00000000..64411c33 --- /dev/null +++ b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AgentDecisionTest.java @@ -0,0 +1,39 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class AgentDecisionTest +{ + @Test + void saysWhoDecidedAndWithWhichVersion() + { + AgentDecision allowed = AgentDecision.allowed(4, 11); + assertThat(allowed.allowed()).isTrue(); + assertThat(allowed.determined()).isTrue(); + assertThat(allowed.policyId()).isEqualTo(11L); + assertThat(allowed.policyVersion()).isEqualTo(4L); + assertThat(allowed).hasToString("ALLOWED by policy 11 at version 4"); + + AgentDecision denied = AgentDecision.denied(4, 12); + assertThat(denied.allowed()).isFalse(); + assertThat(denied.determined()).isTrue(); + assertThat(denied.outcome()).isEqualTo(AgentDecision.Outcome.DENIED); + + AgentDecision open = AgentDecision.notDetermined(4); + assertThat(open.determined()).isFalse(); + assertThat(open.policyId()).isNull(); + assertThat(open).hasToString("NOT_DETERMINED at version 4"); + + AgentDecision none = AgentDecision.withoutSnapshot(); + assertThat(none.outcome()).isEqualTo(AgentDecision.Outcome.NOT_DETERMINED); + assertThat(none.policyVersion()).isNull(); + assertThat(none).hasToString("NOT_DETERMINED"); + } +} diff --git a/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AgentSettingsTest.java b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AgentSettingsTest.java new file mode 100644 index 00000000..94fe35b3 --- /dev/null +++ b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AgentSettingsTest.java @@ -0,0 +1,87 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.nio.file.Path; +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; + +class AgentSettingsTest +{ + private static AgentSettings.Builder valid() + { + return AgentSettings.builder().server(URI.create("https://grantforge.example.com")).token(" gfa_abc ").instance("namenode-1:8020") + .cacheDirectory(Path.of("/var/lib/agent")); + } + + @Test + void defaultsSuitAnAgent() + { + AgentSettings settings = valid().build(); + + assertThat(settings.server()).isEqualTo(URI.create("https://grantforge.example.com")); + assertThat(settings.token()).isEqualTo("gfa_abc"); + assertThat(settings.instance()).isEqualTo("namenode-1:8020"); + assertThat(settings.host()).isNotBlank(); + assertThat(settings.agentVersion()).isEqualTo("unknown"); + assertThat(settings.cacheDirectory()).isEqualTo(Path.of("/var/lib/agent")); + assertThat(settings.connectTimeout()).isEqualTo(Duration.ofSeconds(10)); + assertThat(settings.readTimeout()).isEqualTo(Duration.ofSeconds(30)); + assertThat(settings.refreshInterval()).isEqualTo(Duration.ofSeconds(30)); + assertThat(settings.auditBatchSize()).isEqualTo(500); + assertThat(settings.auditFlushInterval()).isEqualTo(Duration.ofSeconds(5)); + assertThat(settings.auditQueueCapacity()).isEqualTo(10_000); + assertThat(settings.spoolLimitBytes()).isEqualTo(64L * 1024 * 1024); + assertThat(settings.trustedKey()).isNull(); + } + + @Test + void takesWhatIsGiven() + { + SigningKey key = SigningKey.of(FakeServer.publicKey(FakeServer.keyPair())); + AgentSettings settings = valid().host("nn1").agentVersion("2026.0.0").timeouts(Duration.ofSeconds(1), Duration.ofSeconds(2)) + .refreshInterval(Duration.ofSeconds(3)).audit(10, Duration.ofSeconds(4), 20).spoolLimitBytes(0).trustedKey(key).build(); + + assertThat(settings.host()).isEqualTo("nn1"); + assertThat(settings.agentVersion()).isEqualTo("2026.0.0"); + assertThat(settings.connectTimeout()).isEqualTo(Duration.ofSeconds(1)); + assertThat(settings.readTimeout()).isEqualTo(Duration.ofSeconds(2)); + assertThat(settings.refreshInterval()).isEqualTo(Duration.ofSeconds(3)); + assertThat(settings.auditBatchSize()).isEqualTo(10); + assertThat(settings.auditFlushInterval()).isEqualTo(Duration.ofSeconds(4)); + assertThat(settings.auditQueueCapacity()).isEqualTo(20); + assertThat(settings.spoolLimitBytes()).isZero(); + assertThat(settings.trustedKey()).isSameAs(key); + } + + @Test + void refusesWhatCannotWork() + { + assertThatIllegalArgumentException().isThrownBy(() -> AgentSettings.builder().build()).withMessageContaining("server"); + assertThatIllegalArgumentException().isThrownBy(() -> valid().server(URI.create("ftp://host")).build()).withMessageContaining("server"); + assertThatIllegalArgumentException().isThrownBy(() -> valid().server(URI.create("http:///path")).build()).withMessageContaining("server"); + assertThatIllegalArgumentException().isThrownBy(() -> valid().token(" ").build()).withMessageContaining("token"); + assertThatIllegalArgumentException().isThrownBy(() -> valid().instance("name node").build()).withMessageContaining("instance"); + assertThatIllegalArgumentException().isThrownBy(() -> valid().instance("x".repeat(129)).build()).withMessageContaining("instance"); + assertThatIllegalArgumentException().isThrownBy(() -> AgentSettings.builder().server(URI.create("http://h")).token("t").instance("i").build()) + .withMessageContaining("cache directory"); + assertThatIllegalArgumentException().isThrownBy(() -> valid().timeouts(Duration.ZERO, Duration.ofSeconds(1)).build()) + .withMessageContaining("connect timeout"); + assertThatIllegalArgumentException().isThrownBy(() -> valid().timeouts(Duration.ofSeconds(1), Duration.ofSeconds(-1)).build()) + .withMessageContaining("read timeout"); + assertThatIllegalArgumentException().isThrownBy(() -> valid().refreshInterval(Duration.ZERO).build()).withMessageContaining("refresh"); + assertThatIllegalArgumentException().isThrownBy(() -> valid().audit(0, Duration.ofSeconds(1), 10).build()).withMessageContaining("batch"); + assertThatIllegalArgumentException().isThrownBy(() -> valid().audit(1001, Duration.ofSeconds(1), 2000).build()).withMessageContaining("batch"); + assertThatIllegalArgumentException().isThrownBy(() -> valid().audit(10, Duration.ofSeconds(1), 5).build()).withMessageContaining("queue"); + assertThatIllegalArgumentException().isThrownBy(() -> valid().audit(10, Duration.ZERO, 50).build()).withMessageContaining("flush"); + assertThatIllegalArgumentException().isThrownBy(() -> valid().spoolLimitBytes(-1).build()).withMessageContaining("spool"); + } +} diff --git a/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AuditShipperTest.java b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AuditShipperTest.java new file mode 100644 index 00000000..7322ca72 --- /dev/null +++ b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AuditShipperTest.java @@ -0,0 +1,127 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +class AuditShipperTest +{ + @TempDir + Path cache; + + private FakeServer server; + + @BeforeEach + void start() throws IOException + { + server = new FakeServer(); + } + + @AfterEach + void stop() + { + server.close(); + } + + private AuditShipper shipper(long spoolLimit) + { + return shipper(spoolLimit, cache); + } + + private AuditShipper shipper(long spoolLimit, Path directory) + { + AgentSettings settings = server.settings(directory).spoolLimitBytes(spoolLimit).build(); + return new AuditShipper(settings, new ServerClient(settings)); + } + + private static AccessEvent event(String user) + { + return AccessEvent.builder(user, "/data", "read", true).build(); + } + + @Test + void sendsFullBatchesAndTheRestWhenAsked() + { + AuditShipper shipper = shipper(1_000_000); + shipper.record(event("a")); + shipper.record(event("b")); + shipper.record(event("c")); + + shipper.ship(false); + assertThat(server.eventBatches).hasSize(1); + assertThat(shipper.waiting()).isEqualTo(1); + shipper.ship(true); + assertThat(server.eventBatches).hasSize(2); + assertThat(server.eventsReceived()).isEqualTo(3); + assertThat(shipper.waiting()).isZero(); + } + + @Test + void dropsEventsRatherThanWaitWhenTheQueueIsFull() + { + AuditShipper shipper = shipper(1_000_000); + for (int index = 0; index < 4; index++) { + assertThat(shipper.record(event("u" + index))).isTrue(); + } + + assertThat(shipper.record(event("late"))).isFalse(); + assertThat(shipper.dropped()).isEqualTo(1); + } + + @Test + void spoolsWhileTheServerIsAwayAndSendsLater() throws IOException + { + AuditShipper shipper = shipper(1_000_000); + server.down(true); + shipper.record(event("a")); + shipper.record(event("b")); + shipper.record(event("c")); + shipper.ship(true); + + assertThat(shipper.spooled()).hasSize(2); + assertThat(server.eventsReceived()).isZero(); + server.down(false); + shipper.record(event("d")); + shipper.ship(true); + assertThat(server.eventsReceived()).isEqualTo(4); + assertThat(shipper.spooled()).isEmpty(); + assertThat(shipper.dropped()).isZero(); + // A spooled file that cannot be read is dropped instead of blocking the others. + Files.createDirectories(cache.resolve(AuditShipper.SPOOL)); + Files.writeString(cache.resolve(AuditShipper.SPOOL).resolve("0-broken.json"), "not json"); + shipper.ship(true); + assertThat(shipper.spooled()).isEmpty(); + } + + @Test + void keepsTheSpoolUnderItsLimitByDroppingTheOldest() + { + // About one batch of two events fits. + AuditShipper shipper = shipper(800); + server.down(true); + for (int round = 0; round < 3; round++) { + shipper.record(event("a" + round)); + shipper.record(event("b" + round)); + shipper.ship(true); + } + + assertThat(shipper.spooled()).hasSize(1); + assertThat(shipper.dropped()).isEqualTo(4); + AuditShipper none = shipper(0, cache.resolve("none")); + none.record(event("x")); + none.ship(true); + assertThat(none.dropped()).isEqualTo(1); + } +} diff --git a/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/FakeServer.java b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/FakeServer.java new file mode 100644 index 00000000..ad00380e --- /dev/null +++ b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/FakeServer.java @@ -0,0 +1,245 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.MessageDigest; +import java.security.Signature; +import java.util.Base64; +import java.util.HexFormat; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * The agent API of a GrantForge server, as far as agents see it: heartbeats, signed snapshots with ETags, the signing key + * and access events. Signs with a real Ed25519 key from the JDK, like the server does. + */ +final class FakeServer + implements AutoCloseable +{ + static final String TOKEN = "gfa_test-token"; + private static final ObjectMapper JSON = new ObjectMapper(); + + final List heartbeats = new CopyOnWriteArrayList<>(); + final List eventBatches = new CopyOnWriteArrayList<>(); + final AtomicInteger downloads = new AtomicInteger(); + final AtomicInteger keyRequests = new AtomicInteger(); + + private final HttpServer server; + private volatile KeyPair keys = keyPair(); + private volatile byte[] snapshot = Snapshots.json(1, true).getBytes(StandardCharsets.UTF_8); + private volatile long policyVersion = 1; + private volatile boolean down; + private volatile boolean tamper; + private volatile long refreshSeconds = 30; + + FakeServer() throws IOException + { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/api/v1/agent/heartbeat", this::heartbeat); + server.createContext("/api/v1/agent/policies", this::policies); + server.createContext("/api/v1/agent/signing-key", this::signingKey); + server.createContext("/api/v1/agent/access-events", this::accessEvents); + server.start(); + } + + static KeyPair keyPair() + { + try { + return KeyPairGenerator.getInstance("Ed25519").generateKeyPair(); + } + catch (GeneralSecurityException impossible) { + throw new IllegalStateException(impossible); + } + } + + static String sign(KeyPair keys, byte[] data) + { + try { + Signature signature = Signature.getInstance("Ed25519"); + signature.initSign(keys.getPrivate()); + signature.update(data); + return Base64.getEncoder().encodeToString(signature.sign()); + } + catch (GeneralSecurityException impossible) { + throw new IllegalStateException(impossible); + } + } + + static String publicKey(KeyPair keys) + { + return Base64.getEncoder().encodeToString(keys.getPublic().getEncoded()); + } + + /** The server's key id: the first 16 hex digits of the SHA-256 of the X.509 key. */ + static String keyId(KeyPair keys) + { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(keys.getPublic().getEncoded())).substring(0, 16); + } + catch (GeneralSecurityException impossible) { + throw new IllegalStateException(impossible); + } + } + + /** Settings that reach this server, with short timeouts and batches of two events. */ + AgentSettings.Builder settings(java.nio.file.Path cache) + { + return AgentSettings.builder().server(uri()).token(TOKEN).instance("test-agent").host("test-host").agentVersion("test") + .cacheDirectory(cache).timeouts(java.time.Duration.ofSeconds(2), java.time.Duration.ofSeconds(5)) + .refreshInterval(java.time.Duration.ofSeconds(7)).audit(2, java.time.Duration.ofMillis(50), 4); + } + + URI uri() + { + return URI.create("http://127.0.0.1:" + server.getAddress().getPort()); + } + + KeyPair keys() + { + return keys; + } + + void publish(String json, long version) + { + snapshot = json.getBytes(StandardCharsets.UTF_8); + policyVersion = version; + } + + void rotateKey() + { + keys = keyPair(); + } + + void down(boolean value) + { + down = value; + } + + /** Sends snapshots whose body no longer matches the signature. */ + void tamper(boolean value) + { + tamper = value; + } + + void refreshSeconds(long value) + { + refreshSeconds = value; + } + + String etag() + { + return "\"" + Integer.toHexString(java.util.Arrays.hashCode(snapshot)) + "\""; + } + + private boolean refused(HttpExchange exchange) throws IOException + { + if (down) { + respond(exchange, 503, "{\"title\":\"Service Unavailable\"}"); + return true; + } + if (!("Bearer " + TOKEN).equals(exchange.getRequestHeaders().getFirst("Authorization"))) { + respond(exchange, 401, "{\"title\":\"Unauthorized\"}"); + return true; + } + return false; + } + + private void heartbeat(HttpExchange exchange) throws IOException + { + if (refused(exchange)) { + return; + } + heartbeats.add(JSON.readTree(exchange.getRequestBody())); + respond(exchange, 200, "{\"policyVersion\":" + policyVersion + ",\"refreshSeconds\":" + refreshSeconds + "}"); + } + + private void policies(HttpExchange exchange) throws IOException + { + if (refused(exchange)) { + return; + } + String etag = etag(); + if (etag.equals(exchange.getRequestHeaders().getFirst("If-None-Match"))) { + exchange.getResponseHeaders().set("ETag", etag); + exchange.sendResponseHeaders(304, -1); + exchange.close(); + return; + } + downloads.incrementAndGet(); + byte[] body = snapshot; + String signature = sign(keys, body); + if (tamper) { + body = new String(body, StandardCharsets.UTF_8).replace("\"allow\"", "\"deny\"").getBytes(StandardCharsets.UTF_8); + } + exchange.getResponseHeaders().set("ETag", etag); + exchange.getResponseHeaders().set(ServerClient.VERSION_HEADER, Long.toString(policyVersion)); + exchange.getResponseHeaders().set(ServerClient.KEY_HEADER, keyId(keys)); + exchange.getResponseHeaders().set(ServerClient.SIGNATURE_HEADER, signature); + respond(exchange, 200, body); + } + + private void signingKey(HttpExchange exchange) throws IOException + { + if (refused(exchange)) { + return; + } + keyRequests.incrementAndGet(); + respond(exchange, 200, "{\"keyId\":\"" + keyId(keys) + "\",\"algorithm\":\"Ed25519\",\"publicKey\":\"" + publicKey(keys) + "\"}"); + } + + private void accessEvents(HttpExchange exchange) throws IOException + { + if (refused(exchange)) { + return; + } + JsonNode batch = JSON.readTree(exchange.getRequestBody()); + eventBatches.add(batch); + respond(exchange, 200, "{\"accepted\":" + batch.path("events").size() + ",\"duplicates\":0,\"expired\":0}"); + } + + int eventsReceived() + { + int count = 0; + for (JsonNode batch : eventBatches) { + count += batch.path("events").size(); + } + return count; + } + + private static void respond(HttpExchange exchange, int status, String body) throws IOException + { + respond(exchange, status, body.getBytes(StandardCharsets.UTF_8)); + } + + private static void respond(HttpExchange exchange, int status, byte[] body) throws IOException + { + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, body.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(body); + } + } + + @Override + public void close() + { + server.stop(0); + } +} diff --git a/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/GrantForgeAgentTest.java b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/GrantForgeAgentTest.java new file mode 100644 index 00000000..05582637 --- /dev/null +++ b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/GrantForgeAgentTest.java @@ -0,0 +1,200 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Map; + +import static java.util.Objects.requireNonNull; +import static org.assertj.core.api.Assertions.assertThat; + +class GrantForgeAgentTest +{ + @TempDir + Path cache; + + private FakeServer server; + + @BeforeEach + void start() throws IOException + { + server = new FakeServer(); + } + + @AfterEach + void stop() + { + server.close(); + } + + private GrantForgeAgent agent(AgentSettings settings) + { + return new GrantForgeAgent(settings, Map.of()); + } + + private static long version(GrantForgeAgent agent) + { + return requireNonNull(agent.snapshot(), "no snapshot applied").policyVersion(); + } + + private static AgentDecision analystSelects(GrantForgeAgent agent) + { + return agent.decide(Snapshots.request("alice", "select", "sales", "orders")); + } + + @Test + void decidesNothingUntilASnapshotIsApplied() + { + GrantForgeAgent agent = agent(server.settings(cache).build()); + + assertThat(agent.snapshot()).isNull(); + assertThat(analystSelects(agent).outcome()).isEqualTo(AgentDecision.Outcome.NOT_DETERMINED); + assertThat(analystSelects(agent).policyVersion()).isNull(); + } + + @Test + void appliesSnapshotsAsThePolicyVersionChanges() + { + GrantForgeAgent agent = agent(server.settings(cache).build()); + server.refreshSeconds(15); + + assertThat(agent.refresh()).isEqualTo(15); + assertThat(analystSelects(agent).allowed()).isTrue(); + assertThat(server.keyRequests.get()).isEqualTo(1); + assertThat(Files.exists(cache.resolve(SnapshotStore.BODY))).isTrue(); + + agent.refresh(); + assertThat(server.downloads.get()).isEqualTo(1); + assertThat(server.heartbeats.get(1).path("appliedPolicyVersion").asLong()).isEqualTo(1); + + server.publish(Snapshots.json(2, false), 2); + agent.refresh(); + assertThat(version(agent)).isEqualTo(2); + assertThat(analystSelects(agent).outcome()).isEqualTo(AgentDecision.Outcome.NOT_DETERMINED); + assertThat(server.keyRequests.get()).isEqualTo(1); + assertThat(agent.serverReachable()).isTrue(); + } + + @Test + void anUnchangedSnapshotIsNotDownloadedAgain() + { + GrantForgeAgent agent = agent(server.settings(cache).build()); + agent.refresh(); + // The server's version moves while the content stays: the ETag answers 304. + server.publish(Snapshots.json(1, true), 9); + + agent.refresh(); + assertThat(server.downloads.get()).isEqualTo(1); + assertThat(version(agent)).isEqualTo(1); + } + + @Test + void keepsThePoliciesItHasWhenASnapshotIsRejected() + { + GrantForgeAgent agent = agent(server.settings(cache).build()); + agent.refresh(); + + server.tamper(true); + server.publish(Snapshots.json(2, true), 2); + assertThat(agent.refresh()).isEqualTo(7); + assertThat(version(agent)).isEqualTo(1); + + server.tamper(false); + server.publish("{\"format\": 99}", 3); + agent.refresh(); + assertThat(version(agent)).isEqualTo(1); + assertThat(analystSelects(agent).allowed()).isTrue(); + } + + @Test + void followsAKeyTheServerRotatedUnlessOneIsPinned() + { + GrantForgeAgent agent = agent(server.settings(cache).build()); + agent.refresh(); + server.rotateKey(); + server.publish(Snapshots.json(2, true), 2); + + agent.refresh(); + assertThat(version(agent)).isEqualTo(2); + assertThat(server.keyRequests.get()).isEqualTo(2); + + SigningKey other = SigningKey.of(FakeServer.publicKey(FakeServer.keyPair())); + GrantForgeAgent pinned = agent(server.settings(cache.resolve("pinned")).trustedKey(other).build()); + pinned.refresh(); + assertThat(pinned.snapshot()).isNull(); + } + + @Test + void startsWithTheStoredSnapshotWhileTheServerIsAway() + { + agent(server.settings(cache).build()).refresh(); + server.down(true); + + GrantForgeAgent restarted = agent(server.settings(cache).build()); + restarted.loadStored(); + assertThat(analystSelects(restarted).allowed()).isTrue(); + assertThat(restarted.refresh()).isEqualTo(7); + assertThat(restarted.serverReachable()).isFalse(); + restarted.refresh(); + assertThat(version(restarted)).isEqualTo(1); + + GrantForgeAgent pinnedElsewhere = agent(server.settings(cache).trustedKey(SigningKey.of(FakeServer.publicKey(FakeServer.keyPair()))) + .build()); + pinnedElsewhere.loadStored(); + assertThat(pinnedElsewhere.snapshot()).isNull(); + } + + @Test + void anUnreadableStoredSnapshotIsSkipped() throws IOException + { + agent(server.settings(cache).build()).refresh(); + Files.writeString(cache.resolve(SnapshotStore.META), Files.readString(cache.resolve(SnapshotStore.META))); + Files.writeString(cache.resolve(SnapshotStore.BODY), "{\"format\": 99}"); + GrantForgeAgent agent = agent(server.settings(cache).build()); + + agent.loadStored(); + assertThat(agent.snapshot()).isNull(); + } + + @Test + void runsInTheBackgroundAndShipsEventsWhenClosed() throws Exception + { + AgentSettings settings = server.settings(cache).audit(2, Duration.ofSeconds(60), 10).build(); + try (GrantForgeAgent agent = GrantForgeAgent.start(settings, Map.of())) { + long deadline = System.currentTimeMillis() + 10_000; + while (agent.snapshot() == null && System.currentTimeMillis() < deadline) { + Thread.sleep(20); + } + assertThat(analystSelects(agent).allowed()).isTrue(); + assertThat(agent.record(AccessEvent.builder("alice", "sales.orders.id", "select", true).decidedBy(analystSelects(agent)) + .build())).isTrue(); + assertThat(agent.droppedEvents()).isZero(); + } + assertThat(server.eventsReceived()).isEqualTo(1); + assertThat(server.eventBatches.get(0).path("events").get(0).path("enforcer").asText()).isEqualTo("GRANTFORGE"); + } + + @Test + void shipsEventsOnItsOwnEveryFlushInterval() throws Exception + { + try (GrantForgeAgent agent = GrantForgeAgent.start(server.settings(cache).build(), Map.of())) { + agent.record(AccessEvent.builder("bob", "/data", "read", false).build()); + long deadline = System.currentTimeMillis() + 10_000; + while (server.eventsReceived() == 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(20); + } + assertThat(server.eventsReceived()).isEqualTo(1); + } + } +} diff --git a/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/ServerClientTest.java b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/ServerClientTest.java new file mode 100644 index 00000000..7ac23931 --- /dev/null +++ b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/ServerClientTest.java @@ -0,0 +1,132 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import com.fasterxml.jackson.databind.JsonNode; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class ServerClientTest +{ + @TempDir + Path cache; + + private FakeServer server; + private ServerClient client; + + @BeforeEach + void start() throws IOException + { + server = new FakeServer(); + client = new ServerClient(server.settings(cache).build()); + } + + @AfterEach + void stop() + { + server.close(); + } + + @Test + void sendsHeartbeatsWithTheToken() throws IOException + { + server.refreshSeconds(12); + + ServerClient.Heartbeat heartbeat = client.heartbeat(null); + assertThat(heartbeat.policyVersion()).isEqualTo(1); + assertThat(heartbeat.refreshSeconds()).isEqualTo(12); + client.heartbeat(5L); + JsonNode first = server.heartbeats.get(0); + assertThat(first.path("instance").asText()).isEqualTo("test-agent"); + assertThat(first.path("host").asText()).isEqualTo("test-host"); + assertThat(first.path("agentVersion").asText()).isEqualTo("test"); + assertThat(first.path("appliedPolicyVersion").isNull()).isTrue(); + assertThat(server.heartbeats.get(1).path("appliedPolicyVersion").asLong()).isEqualTo(5); + } + + @Test + void downloadsSnapshotsUnlessTheAgentHasThem() throws IOException + { + ServerClient.Download download = client.policies(null); + + assertThat(download.changed()).isTrue(); + assertThat(new String(download.body(), StandardCharsets.UTF_8)).contains("\"warehouse\""); + assertThat(download.etag()).isEqualTo(server.etag()); + assertThat(download.keyId()).isEqualTo(FakeServer.keyId(server.keys())); + assertThat(SigningKey.of(FakeServer.publicKey(server.keys())).verifies(download.body(), download.signature())).isTrue(); + assertThat(client.policies(download.etag()).changed()).isFalse(); + assertThat(server.downloads.get()).isEqualTo(1); + } + + @Test + void fetchesTheSigningKey() throws IOException + { + assertThat(client.signingKey().keyId()).isEqualTo(FakeServer.keyId(server.keys())); + } + + @Test + void sendsEventsAndCountsWhatWasStored() throws IOException + { + Map event = AccessEvent.builder("alice", "r", "select", true).build().fields(); + + assertThat(client.send(List.of(event, event))).isEqualTo(2); + JsonNode batch = server.eventBatches.get(0); + assertThat(batch.path("instance").asText()).isEqualTo("test-agent"); + assertThat(batch.path("events").get(0).path("user").asText()).isEqualTo("alice"); + } + + @Test + void reportsRefusalsAndUnreachableServers() throws IOException + { + server.down(true); + assertThatThrownBy(() -> client.heartbeat(null)).isInstanceOf(IOException.class).hasMessageContaining("503") + .hasMessageContaining("Service Unavailable"); + ServerClient wrongToken = new ServerClient(server.settings(cache).token("gfa_wrong").build()); + server.down(false); + assertThatThrownBy(() -> wrongToken.policies(null)).isInstanceOf(IOException.class).hasMessageContaining("401"); + server.close(); + assertThatThrownBy(() -> client.heartbeat(null)).isInstanceOf(IOException.class); + } + + @Test + void refusesAnswersItCannotUse() throws IOException + { + HttpServer odd = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + odd.createContext("/", exchange -> { + byte[] body = "[1]".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(body); + } + }); + odd.start(); + try { + // The address may end in a slash or not. + ServerClient oddClient = new ServerClient(server.settings(cache) + .server(URI.create("http://127.0.0.1:" + odd.getAddress().getPort() + "/")).build()); + assertThatThrownBy(() -> oddClient.heartbeat(null)).isInstanceOf(IOException.class).hasMessageContaining("JSON object"); + assertThatThrownBy(() -> oddClient.policies(null)).isInstanceOf(IOException.class).hasMessageContaining("signature"); + } + finally { + odd.stop(0); + } + } +} diff --git a/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/SigningKeyTest.java b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/SigningKeyTest.java new file mode 100644 index 00000000..1917c5c1 --- /dev/null +++ b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/SigningKeyTest.java @@ -0,0 +1,57 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.util.Base64; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; + +class SigningKeyTest +{ + private final KeyPair keys = FakeServer.keyPair(); + private final byte[] data = "{\"format\":1}".getBytes(StandardCharsets.UTF_8); + + @Test + void checksSignaturesTheJdkMade() + { + SigningKey key = SigningKey.of(FakeServer.publicKey(keys)); + String signature = FakeServer.sign(keys, data); + + assertThat(key.verifies(data, signature)).isTrue(); + assertThat(key.verifies("{\"format\":2}".getBytes(StandardCharsets.UTF_8), signature)).isFalse(); + assertThat(key.verifies(data, FakeServer.sign(FakeServer.keyPair(), data))).isFalse(); + assertThat(key.verifies(data, "not base64!")).isFalse(); + assertThat(key.verifies(data, Base64.getEncoder().encodeToString(new byte[10]))).isFalse(); + } + + @Test + void namesTheKeyAsTheServerDoes() + { + String encoded = FakeServer.publicKey(keys); + SigningKey key = SigningKey.of(" " + encoded + "\n"); + + assertThat(key.keyId()).isEqualTo(FakeServer.keyId(keys)).hasSize(16); + assertThat(key.encoded()).isEqualTo(encoded); + } + + @Test + void refusesWhatIsNotAnEd25519Key() throws Exception + { + KeyPair rsa = KeyPairGenerator.getInstance("RSA").generateKeyPair(); + + assertThatIllegalArgumentException().isThrownBy(() -> SigningKey.of("%%%")).withMessageContaining("X.509"); + assertThatIllegalArgumentException().isThrownBy(() -> SigningKey.of(Base64.getEncoder().encodeToString(new byte[12]))) + .withMessageContaining("X.509"); + assertThatIllegalArgumentException().isThrownBy(() -> SigningKey.of(Base64.getEncoder().encodeToString(rsa.getPublic().getEncoded()))) + .withMessageContaining("not an Ed25519"); + } +} diff --git a/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/SnapshotStoreTest.java b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/SnapshotStoreTest.java new file mode 100644 index 00000000..1d1fe37b --- /dev/null +++ b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/SnapshotStoreTest.java @@ -0,0 +1,79 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyPair; + +import static org.assertj.core.api.Assertions.assertThat; + +class SnapshotStoreTest +{ + @TempDir + Path directory; + + private final KeyPair keys = FakeServer.keyPair(); + private final SigningKey key = SigningKey.of(FakeServer.publicKey(keys)); + + private ServerClient.Download download(String json) + { + byte[] body = json.getBytes(StandardCharsets.UTF_8); + return new ServerClient.Download(body, "\"e1\"", key.keyId(), FakeServer.sign(keys, body)); + } + + @Test + void keepsTheLastSnapshotWithItsKey() throws IOException + { + SnapshotStore store = new SnapshotStore(directory.resolve("nested")); + assertThat(store.load(null)).isNull(); + + store.save(download("{\"v\":1}"), key); + store.save(download("{\"v\":2}"), key); + SnapshotStore.Stored stored = store.load(null); + + assertThat(stored).isNotNull(); + assertThat(new String(stored.snapshot().body(), StandardCharsets.UTF_8)).isEqualTo("{\"v\":2}"); + assertThat(stored.snapshot().etag()).isEqualTo("\"e1\""); + assertThat(stored.key().keyId()).isEqualTo(key.keyId()); + assertThat(store.load(key)).isNotNull(); + assertThat(directory.resolve("nested")).isDirectoryNotContaining("glob:**.tmp"); + } + + @Test + void ignoresASnapshotThatWasChangedOrSignedByAnotherKey() throws IOException + { + SnapshotStore store = new SnapshotStore(directory); + store.save(download("{\"v\":1}"), key); + + assertThat(store.load(SigningKey.of(FakeServer.publicKey(FakeServer.keyPair())))).isNull(); + Files.write(directory.resolve(SnapshotStore.BODY), "{\"v\":9}".getBytes(StandardCharsets.UTF_8)); + assertThat(store.load(null)).isNull(); + } + + @Test + void ignoresBrokenOrIncompleteMetadata() throws IOException + { + SnapshotStore store = new SnapshotStore(directory); + store.save(download("{\"v\":1}"), key); + Path meta = directory.resolve(SnapshotStore.META); + String original = Files.readString(meta); + + Files.writeString(meta, original.replaceAll("(?m)^signingKey=.*$", "signingKey=AAAA")); + assertThat(store.load(null)).isNull(); + Files.writeString(meta, original.replaceAll("(?m)^keyId=.*$", "keyId=0000000000000000")); + assertThat(store.load(null)).isNull(); + Files.writeString(meta, original.replaceAll("(?m)^etag=.*$", "")); + assertThat(store.load(null)).isNull(); + Files.delete(meta); + assertThat(store.load(null)).isNull(); + } +} diff --git a/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/SnapshotTest.java b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/SnapshotTest.java new file mode 100644 index 00000000..0c2ebbaf --- /dev/null +++ b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/SnapshotTest.java @@ -0,0 +1,153 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import org.devlive.grantforge.policy.engine.AccessRequest; +import org.devlive.grantforge.policy.engine.ConditionEvaluator; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; + +class SnapshotTest +{ + /** Holds when the client address starts with one of the values' first octet, enough for the test's 10.0.0.0/8. */ + private static final ConditionEvaluator FIRST_OCTET = (values, request) -> { + Object ip = request.context().get("clientIp"); + return ip != null && values.stream().anyMatch(range -> ip.toString().startsWith(range.substring(0, range.indexOf('.') + 1))); + }; + + private static Snapshot parse(String json, Map evaluators) + { + return Snapshot.parse(json.getBytes(StandardCharsets.UTF_8), evaluators); + } + + private static Snapshot snapshot() + { + return parse(Snapshots.json(7, true), Map.of("ip-range", FIRST_OCTET)); + } + + @Test + void readsTheServiceAndCountsPolicies() + { + Snapshot snapshot = snapshot(); + + assertThat(snapshot.service()).isEqualTo("warehouse"); + assertThat(snapshot.serviceType()).isEqualTo("hive"); + assertThat(snapshot.serviceEnabled()).isTrue(); + assertThat(snapshot.policyVersion()).isEqualTo(7); + assertThat(snapshot.accessPolicies()).isEqualTo(3); + assertThat(snapshot.otherPolicies()).isEqualTo(1); + assertThat(snapshot.rolesOf("alice")).containsExactly("analyst"); + assertThat(snapshot.groupsOf("bob")).containsExactly("ops", "spies"); + assertThat(snapshot.rolesOf("nobody")).isEmpty(); + } + + @Test + void decidesWithTheRolesAndGroupsTheSnapshotGives() + { + Snapshot snapshot = snapshot(); + + AgentDecision analyst = snapshot.decide(Snapshots.request("alice", "select", "sales", "orders")); + assertThat(analyst.outcome()).isEqualTo(AgentDecision.Outcome.ALLOWED); + assertThat(analyst.policyId()).isEqualTo(11L); + assertThat(analyst.policyVersion()).isEqualTo(7L); + assertThat(snapshot.decide(Snapshots.request("bob", "select", "sales", "orders")).allowed()).isTrue(); + AgentDecision spy = snapshot.decide(Snapshots.request("bob", "update", "sales", "orders")); + assertThat(spy.outcome()).isEqualTo(AgentDecision.Outcome.DENIED); + assertThat(spy.policyId()).isEqualTo(11L); + assertThat(snapshot.decide(Snapshots.request("mallory", "select", "sales", "orders")).outcome()) + .isEqualTo(AgentDecision.Outcome.NOT_DETERMINED); + // Groups the system knows count as well as those the snapshot gives. + AccessRequest fromSystem = AccessRequest.builder("zed", "select").groups("ops").resource("database", "sales") + .resource("table", "t").resource("column", "c").build(); + assertThat(snapshot.decide(fromSystem).allowed()).isTrue(); + } + + @Test + void appliesConditionsThroughTheAgentsEvaluators() + { + Snapshot snapshot = snapshot(); + AccessRequest inside = AccessRequest.builder("carol", "select").resource("database", "hr").resource("table", "salaries") + .resource("column", "pay").context(Map.of("clientIp", "10.1.2.3")).build(); + AccessRequest outside = AccessRequest.builder("carol", "select").resource("database", "hr").resource("table", "salaries") + .resource("column", "pay").context(Map.of("clientIp", "192.168.1.1")).build(); + + assertThat(snapshot.decide(inside).outcome()).isEqualTo(AgentDecision.Outcome.DENIED); + assertThat(snapshot.decide(outside).allowed()).isTrue(); + // Without the evaluator the deny holds, the safe direction. + assertThat(parse(Snapshots.json(7, true), Map.of()).decide(outside).outcome()).isEqualTo(AgentDecision.Outcome.DENIED); + } + + @Test + void expiredPoliciesDecideNothing() + { + AccessRequest archive = AccessRequest.builder("dave", "select").resource("database", "archive").resource("table", "t") + .resource("column", "c").time(Instant.parse("2026-01-01T00:00:00Z")).build(); + + assertThat(snapshot().decide(archive).outcome()).isEqualTo(AgentDecision.Outcome.NOT_DETERMINED); + } + + @Test + void aServiceNotInUseDecidesNothing() + { + Snapshot disabled = parse(Snapshots.json(3, false), Map.of()); + + AgentDecision decision = disabled.decide(Snapshots.request("alice", "select", "sales", "orders")); + assertThat(decision.outcome()).isEqualTo(AgentDecision.Outcome.NOT_DETERMINED); + assertThat(decision.policyVersion()).isEqualTo(3L); + } + + @Test + void refusesWhatItCannotRead() + { + String valid = Snapshots.json(1, true); + assertThatIllegalArgumentException().isThrownBy(() -> parse("not json", Map.of())).withMessageContaining("JSON"); + assertThatIllegalArgumentException().isThrownBy(() -> parse("[]", Map.of())).withMessageContaining("object"); + assertThatIllegalArgumentException().isThrownBy(() -> parse(valid.replace("\"format\": 1", "\"format\": 2"), Map.of())) + .withMessageContaining("upgrade"); + assertThatIllegalArgumentException().isThrownBy(() -> parse(valid.replace("\"WILDCARD\", \"caseSensitive\": false}],", + "\"FUZZY\", \"caseSensitive\": false}],"), Map.of())).withMessageContaining("FUZZY"); + assertThatIllegalArgumentException().isThrownBy(() -> parse(valid.replace("\"parent\": \"table\"", "\"parent\": \"nowhere\""), Map.of())) + .withMessageContaining("unknown parents"); + assertThatIllegalArgumentException().isThrownBy(() -> parse(valid.replace("\"id\": \"11\"", "\"id\": \"eleven\""), Map.of())) + .withMessageContaining("eleven"); + assertThatIllegalArgumentException().isThrownBy(() -> parse(valid.replace("\"until\": \"2020-01-01T00:00:00Z\"", "\"until\": \"soon\""), + Map.of())).withMessageContaining("until"); + assertThatIllegalArgumentException().isThrownBy(() -> parse(valid.replace("\"policyVersion\": 1", "\"policyVersion\": \"1\""), Map.of())) + .withMessageContaining("policyVersion"); + assertThatIllegalArgumentException().isThrownBy(() -> parse(valid.replace("\"serviceEnabled\": true", "\"serviceEnabled\": 1"), Map.of())) + .withMessageContaining("serviceEnabled"); + assertThatIllegalArgumentException().isThrownBy(() -> parse(valid.replace("\"service\": \"warehouse\",", ""), Map.of())) + .withMessageContaining("service is missing"); + assertThatIllegalArgumentException().isThrownBy(() -> parse(valid.replace("\"roles\": {\"analyst\": [\"alice\"]}", "\"roles\": []"), Map.of())) + .withMessageContaining("roles"); + assertThatIllegalArgumentException().isThrownBy(() -> parse(valid.replace("\"analyst\": [\"alice\"]", "\"analyst\": \"alice\""), Map.of())) + .withMessageContaining("analyst"); + assertThatIllegalArgumentException().isThrownBy(() -> parse(valid.replace("\"analyst\": [\"alice\"]", "\"analyst\": [1]"), Map.of())) + .withMessageContaining("analyst"); + assertThatIllegalArgumentException().isThrownBy(() -> parse(valid.replace("\"groups\": [\"ops\"]", "\"groups\": [1]"), Map.of())) + .withMessageContaining("groups"); + assertThatIllegalArgumentException().isThrownBy(() -> parse(valid.replace("\"definition\": {", "\"definition\": [], \"x\": {"), + Map.of())).withMessageContaining("definition"); + assertThatIllegalArgumentException().isThrownBy(() -> parse(valid.replace("\"policies\": [", "\"policies\": {\"x\": ["), Map.of())); + } + + @Test + void missingRolesAndGroupsMeanNone() + { + String bare = Snapshots.json(1, true).replace("\"roles\": {\"analyst\": [\"alice\"]},", "\"roles\": null,") + .replace(",\n \"groups\": {\"ops\": [\"bob\"], \"spies\": [\"eve\", \"bob\"]}", ""); + + Snapshot snapshot = parse(bare, Map.of()); + assertThat(snapshot.rolesOf("alice")).isEmpty(); + assertThat(snapshot.groupsOf("bob")).isEmpty(); + } +} diff --git a/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/Snapshots.java b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/Snapshots.java new file mode 100644 index 00000000..0dff1548 --- /dev/null +++ b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/Snapshots.java @@ -0,0 +1,74 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.agent; + +import org.devlive.grantforge.policy.engine.AccessRequest; + +/** Snapshots as the server writes them, of a small Hive-like service. */ +final class Snapshots +{ + private Snapshots() + { + } + + /** + * A snapshot: analysts and ops may select in sales; spies may not update there; nobody selects hr salaries from + * 10.x (an override, through the {@code ip-range} evaluator); an archive policy expired in 2020; one masking policy. + */ + static String json(long version, boolean enabled) + { + return """ + {"format": 1, "service": "warehouse", "serviceType": "hive", "typeVersion": 1, "serviceEnabled": %s, + "policyVersion": %d, + "definition": { + "resources": [ + {"name": "table", "parent": "database", "matcher": "WILDCARD", "caseSensitive": false}, + {"name": "column", "parent": "table", "matcher": "WILDCARD", "caseSensitive": false}, + {"name": "database", "parent": null, "matcher": "WILDCARD", "caseSensitive": false}], + "accessTypes": [ + {"name": "select", "impliedGrants": []}, + {"name": "update", "impliedGrants": []}, + {"name": "all", "impliedGrants": ["select", "update"]}], + "conditions": [{"name": "ip", "evaluator": "ip-range", "options": {}}], + "maskTypes": [{"name": "MASK", "transformer": null}]}, + "policies": [ + {"id": "11", "name": "sales", "type": "ACCESS", "priority": "NORMAL", "document": { + "resources": {"database": {"values": ["sales"], "excludes": false, "recursive": false}, + "table": {"values": ["*"], "excludes": false, "recursive": false}, + "column": {"values": ["*"], "excludes": false, "recursive": false}}, + "allow": [{"users": [], "groups": ["ops"], "roles": ["analyst"], "accessTypes": ["select"], "conditions": [], + "maskType": null, "maskValue": null, "rowFilter": null}], + "allowExceptions": [], + "deny": [{"users": [], "groups": ["spies"], "roles": [], "accessTypes": ["update"], "conditions": []}], + "denyExceptions": [], "validity": []}}, + {"id": "12", "name": "no salaries from 10.x", "type": "ACCESS", "priority": "OVERRIDE", "document": { + "resources": {"database": {"values": ["hr"], "excludes": false, "recursive": false}, + "table": {"values": ["salaries"], "excludes": false, "recursive": false}, + "column": {"values": ["*"], "excludes": false, "recursive": false}}, + "allow": [{"users": ["carol"], "groups": [], "roles": [], "accessTypes": ["all"]}], + "deny": [{"users": [], "groups": ["public"], "roles": [], "accessTypes": ["select"], + "conditions": [{"type": "ip", "values": ["10.0.0.0/8"]}]}]}}, + {"id": "13", "name": "mask ssn", "type": "DATA_MASK", "priority": "NORMAL", "document": { + "resources": {"database": {"values": ["sales"], "excludes": false, "recursive": false}}, + "allow": [{"users": [], "groups": ["public"], "roles": [], "accessTypes": ["select"], "maskType": "MASK"}]}}, + {"id": "14", "name": "archive", "type": "ACCESS", "priority": "NORMAL", "document": { + "resources": {"database": {"values": ["archive"], "excludes": false, "recursive": false}, + "table": {"values": ["*"], "excludes": false, "recursive": false}, + "column": {"values": ["*"], "excludes": false, "recursive": false}}, + "allow": [{"users": ["dave"], "groups": [], "roles": [], "accessTypes": ["select"]}], + "validity": [{"from": null, "until": "2020-01-01T00:00:00Z"}]}}], + "roles": {"analyst": ["alice"]}, + "groups": {"ops": ["bob"], "spies": ["eve", "bob"]}} + """.formatted(enabled, version); + } + + /** A request for a column. */ + static AccessRequest request(String user, String accessType, String database, String table) + { + return AccessRequest.builder(user, accessType).resource("database", database).resource("table", table).resource("column", "id") + .build(); + } +} diff --git a/docs/content/architecture/plugins.md b/docs/content/architecture/plugins.md index 11b52f94..3c90e1c3 100644 --- a/docs/content/architecture/plugins.md +++ b/docs/content/architecture/plugins.md @@ -123,6 +123,26 @@ providers: 代理用 `grantforge-policy-engine`（Java 8 API，可以嵌入较老的系统）在本地求值，不必每次访问都调用 GrantForge。 +代理不必自己实现这些协议，`grantforge-agent-core`（Java 8）已经封装好： + +```java +AgentSettings settings = AgentSettings.builder() + .server(URI.create("https://grantforge.example.com")) + .token(System.getenv("GRANTFORGE_AGENT_TOKEN")) + .instance("namenode-1:8020") + .cacheDirectory(Paths.get("/var/lib/grantforge-agent")) + .build(); +GrantForgeAgent agent = GrantForgeAgent.start(settings, evaluators); // 条件求值器，按名称 + +AgentDecision decision = agent.decide(AccessRequest.builder(user, "read").groups(groups).resource("path", path).build()); +agent.record(AccessEvent.builder(user, path, "read", allowed).decidedBy(decision).action("open").build()); +``` + +- 按服务端要求的间隔发心跳；策略版本变化时下载快照（ETag 未变则 304），用服务端的 Ed25519 公钥验签（可在设置中固定公钥，否则首次从服务端获取并保留），校验通过才替换，并保存到缓存目录；服务端不可达时启动沿用最后一份快照。 +- 快照把角色与组展开到用户，`decide` 会把用户在快照中的角色和组加到请求上。服务停用或尚无快照时结果为 `NOT_DETERMINED`，由代理决定回退到系统自身的检查还是拒绝。 +- 访问事件进入有界队列（满了就丢弃并计数，绝不阻塞系统），按批发送；服务端不可达时写入缓存目录下的 `audit-spool/`，恢复后补发，超过上限丢弃最旧的。 +- 依赖 Jackson 2 与 Bouncy Castle（JDK 15 之前没有 Ed25519）；目标系统自带这些库的其他版本，代理打包时需要用 shade 重定位。 + ## 示例 `plugins/grantforge-plugin-example` 是一个完整的插件：类型 `example`（database → table → column 与 path），访问类型 select、update、all，列脱敏、表行过滤、IP 范围条件，配置 url、timeout、password（密码为 `example` 时测试连接成功），并能查找示例库表。全栈端到端测试用它走完“添加服务 → 写策略 → 签发令牌 → 代理拉取 → 访问审计”。 diff --git a/pom.xml b/pom.xml index c69971f3..99ac6a3d 100644 --- a/pom.xml +++ b/pom.xml @@ -24,6 +24,7 @@ core/grantforge-persistence core/grantforge-plugin-api core/grantforge-policy-engine + core/grantforge-agent-core core/grantforge-audit core/grantforge-identity core/grantforge-authz @@ -213,6 +214,11 @@ grantforge-policy-engine ${project.version} + + org.devlive.grantforge + grantforge-agent-core + ${project.version} + org.devlive.grantforge grantforge-service diff --git a/script/ci/coverage_thresholds.txt b/script/ci/coverage_thresholds.txt index 73ed7886..9d36ca63 100644 --- a/script/ci/coverage_thresholds.txt +++ b/script/ci/coverage_thresholds.txt @@ -9,3 +9,4 @@ core/grantforge-authz line=90 branch=85 # decides who may do what; authorization modules hold a higher bar core/grantforge-policy-engine line=90 branch=85 # decides access inside protected systems; authorization modules hold a higher bar core/grantforge-oauth line=90 branch=85 # issues the tokens applications trust; held to the authorization modules' bar +core/grantforge-agent-core line=90 branch=85 # decides access inside protected systems with the engine; authorization modules hold a higher bar From b6cd8ecefa974bacaa2694a8844a38bce2b29865 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 01:32:06 -0400 Subject: [PATCH 02/22] perf: page accounts along an index and look the console up once At a million accounts every user list page sorted the whole tenant and counted it again. An index on tenant, creation time and id lets the database walk the newest accounts, and the search first reads up to 1000 matches unordered: that many or fewer are ordered in memory and counted for free, so a rare match no longer runs the ordered scan through the whole table. The console application's id is kept per catalog version, which application changes now raise, removing a query from every API call. --- .../application/AuthorizationEvaluator.java | 35 +++++++++++- .../grantforge/authz/domain/Application.java | 6 +- .../application/UserAdminService.java | 7 ++- .../grantforge/identity/domain/UserPage.java | 25 ++++++++ .../identity/domain/UserSearchRepository.java | 13 +++++ .../domain/UserSearchRepositoryImpl.java | 57 ++++++++++++++++++- .../main/resources/db/changelog/identity.yaml | 15 +++++ .../identity/domain/UserPageTest.java | 29 ++++++++++ .../domain/UserSearchRepositoryTest.java | 34 +++++++++++ 9 files changed, 213 insertions(+), 8 deletions(-) create mode 100644 core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/domain/UserPage.java create mode 100644 core/grantforge-identity/src/test/java/org/devlive/grantforge/identity/domain/UserPageTest.java diff --git a/core/grantforge-authz/src/main/java/org/devlive/grantforge/authz/application/AuthorizationEvaluator.java b/core/grantforge-authz/src/main/java/org/devlive/grantforge/authz/application/AuthorizationEvaluator.java index 8457254c..76d15fc9 100644 --- a/core/grantforge-authz/src/main/java/org/devlive/grantforge/authz/application/AuthorizationEvaluator.java +++ b/core/grantforge-authz/src/main/java/org/devlive/grantforge/authz/application/AuthorizationEvaluator.java @@ -67,6 +67,9 @@ public final class AuthorizationEvaluator // Loading an application's catalog and preparing its derivation grows with its resources (a hundred thousand // take most of a second), so snapshots share one per application until the catalog counter moves. private final Cache catalogs = Caffeine.newBuilder().maximumSize(64).build(); + // Written once per catalog version and read by every request; a stale read only costs one lookup. + @SuppressWarnings("PMD.AvoidUsingVolatile") + private volatile @Nullable ConsoleId console; private final RoleRepository roles; private final RoleParentRepository parents; private final ResourceRepository resources; @@ -135,14 +138,13 @@ private AuthorizationSnapshot snapshotOf(long accountId, @Nullable Long requeste { return requireNonNull(transactions.execute(status -> { Instant now = clock.instant(); - long applicationId = requested != null ? requested - : applications.findByCode(Application.CONSOLE).map(Application::requireId).orElse(-1L); OptionalLong tenant = TenantContext.currentTenantId(); if (tenant.isEmpty()) { - return compute(accountId, applicationId, now, null).snapshot(); + return compute(accountId, requested != null ? requested : consoleLookup(), now, null).snapshot(); } // The counters are read first: a change committed meanwhile raises them, so the next request recomputes. AuthorizationVersions.Versions current = versions.current(tenant.getAsLong()); + long applicationId = requested != null ? requested : consoleId(current.catalog()); CacheKey key = new CacheKey(tenant.getAsLong(), accountId, applicationId); Cached cached = cache.getIfPresent(key); if (cached != null && cached.versions().equals(current) && now.isBefore(cached.validUntil())) { @@ -154,6 +156,28 @@ private AuthorizationSnapshot snapshotOf(long accountId, @Nullable Long requeste })); } + /** + * The console application's ID, looked up once per catalog version: every API call asks for the console snapshot, + * and application changes raise the catalog version (B-076). + */ + private long consoleId(long catalogVersion) + { + ConsoleId known = console; + if (known != null && known.catalogVersion() == catalogVersion) { + return known.id(); + } + long id = consoleLookup(); + if (id > 0) { + console = new ConsoleId(catalogVersion, id); + } + return id; + } + + private long consoleLookup() + { + return applications.findByCode(Application.CONSOLE).map(Application::requireId).orElse(-1L); + } + /** * Works out a snapshot in an application, and until when time alone leaves it valid; within a transaction. * @@ -428,6 +452,11 @@ Map usable(List roles, List applying, Insta } } + /** The console application's ID as of a catalog version. */ + private record ConsoleId(long catalogVersion, long id) + { + } + /** Whose snapshot: an account of a tenant, in an application. */ private record CacheKey(long tenantId, long accountId, long applicationId) { diff --git a/core/grantforge-authz/src/main/java/org/devlive/grantforge/authz/domain/Application.java b/core/grantforge-authz/src/main/java/org/devlive/grantforge/authz/domain/Application.java index 06ddff2e..050a5259 100644 --- a/core/grantforge-authz/src/main/java/org/devlive/grantforge/authz/domain/Application.java +++ b/core/grantforge-authz/src/main/java/org/devlive/grantforge/authz/domain/Application.java @@ -7,8 +7,10 @@ import jakarta.persistence.Column; import jakarta.persistence.Entity; +import jakarta.persistence.EntityListeners; import jakarta.persistence.Table; import org.devlive.grantforge.common.lang.Strings; +import org.devlive.grantforge.persistence.authz.AuthorizationChangeListener; import org.devlive.grantforge.persistence.entity.BaseEntity; import org.jspecify.annotations.Nullable; @@ -17,10 +19,12 @@ /** * Something whose resources are authorized, such as the GrantForge console itself. The catalog is shared by all - * tenants (D-40): resources describe the software, roles and grants (per tenant) refer to them. + * tenants (D-40): resources describe the software, roles and grants (per tenant) refer to them. Changes raise the + * catalog version like resource changes do, so caches keyed by it, such as the console's ID, follow them (D-86). */ @Entity @Table(name = "gf_application") +@EntityListeners(AuthorizationChangeListener.class) public class Application extends BaseEntity { diff --git a/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/application/UserAdminService.java b/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/application/UserAdminService.java index 2ba5301c..e495626c 100644 --- a/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/application/UserAdminService.java +++ b/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/application/UserAdminService.java @@ -26,6 +26,7 @@ import org.devlive.grantforge.identity.domain.UserAccount; import org.devlive.grantforge.identity.domain.UserAccountRepository; import org.devlive.grantforge.identity.domain.UserCriteria; +import org.devlive.grantforge.identity.domain.UserPage; import org.devlive.grantforge.identity.domain.UserRow; import org.devlive.grantforge.persistence.authz.AuthorizationChanges; import org.devlive.grantforge.persistence.query.InClauseBatcher; @@ -153,9 +154,9 @@ PageResult search(long actorId, UserFilter filter, PageQuery page, return requireNonNull(transactions.execute(status -> { UserCriteria criteria = criteria(actorId, filter); Specification scope = scopes.scope(actorId, UserAccount.class, action); - List items = accounts.search(criteria, scope, now, page.offset(), page.size()).stream() - .map(row -> UserSummary.from(row, now)).toList(); - return new PageResult<>(items, page.page(), page.size(), accounts.count(criteria, scope, now)); + UserPage found = accounts.page(criteria, scope, now, page.offset(), page.size()); + List items = found.rows().stream().map(row -> UserSummary.from(row, now)).toList(); + return new PageResult<>(items, page.page(), page.size(), found.total()); })); } diff --git a/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/domain/UserPage.java b/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/domain/UserPage.java new file mode 100644 index 00000000..7ae90f7a --- /dev/null +++ b/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/domain/UserPage.java @@ -0,0 +1,25 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.identity.domain; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** + * A page of matching accounts with the number of all matches. + * + * @param rows the accounts of the page, the newest first + * @param total how many accounts match + */ +public record UserPage(List rows, long total) +{ + /** Copies the rows. */ + public UserPage + { + rows = List.copyOf(requireNonNull(rows, "rows")); + } +} diff --git a/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/domain/UserSearchRepository.java b/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/domain/UserSearchRepository.java index 406b8c5d..70499099 100644 --- a/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/domain/UserSearchRepository.java +++ b/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/domain/UserSearchRepository.java @@ -25,6 +25,19 @@ public interface UserSearchRepository */ List search(UserCriteria criteria, Specification scope, Instant now, long offset, int limit); + /** + * Lists a page of matching accounts, the newest first, with the number of all matches; cheaper than + * {@link #search} and {@link #count}, which both read the matches. + * + * @param criteria the filters + * @param scope the accounts the reader may see, combined with the filters + * @param now the current time, to tell locked accounts apart + * @param offset how many matches to skip + * @param limit how many matches to return at most + * @return the accounts and the total + */ + UserPage page(UserCriteria criteria, Specification scope, Instant now, long offset, int limit); + /** * Counts matching accounts. * diff --git a/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/domain/UserSearchRepositoryImpl.java b/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/domain/UserSearchRepositoryImpl.java index de227ff5..033e016e 100644 --- a/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/domain/UserSearchRepositoryImpl.java +++ b/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/domain/UserSearchRepositoryImpl.java @@ -6,6 +6,7 @@ package org.devlive.grantforge.identity.domain; import jakarta.persistence.EntityManager; +import jakarta.persistence.Tuple; import jakarta.persistence.criteria.CriteriaBuilder; import jakarta.persistence.criteria.CriteriaQuery; import jakarta.persistence.criteria.Path; @@ -18,7 +19,9 @@ import java.time.Instant; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; +import java.util.Optional; import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; @@ -26,6 +29,11 @@ /** * Builds the account search from only the filters that are set, combined with the reader's data scope: the page of ids * comes from a criteria query, the rows with the primary department from one DTO projection. + * + *

The matches are first read unordered, up to {@value #SMALL} of them (D-86). As many or fewer are ordered here, and + * their number is the total: a rare match would otherwise send the database along the creation-order index through + * the whole table, and the total would take a second full scan. Beyond that the match is common, the ordered page is + * found early along the index, and the total is counted. */ final class UserSearchRepositoryImpl implements UserSearchRepository @@ -37,6 +45,12 @@ final class UserSearchRepositoryImpl + " left join OrgUnit o on o.id = m.orgUnitId" + " where a.id in :ids"; + /** Most matches ordered in memory instead of by the database. */ + static final int SMALL = 1000; + + private static final Comparator NEWEST_FIRST = Comparator.comparing(match -> match.get(1, Instant.class)) + .thenComparing(match -> match.get(0, Long.class)).reversed(); + private final EntityManager entities; /** @@ -51,13 +65,54 @@ final class UserSearchRepositoryImpl @Override public List search(UserCriteria criteria, Specification scope, Instant now, long offset, int limit) + { + return rows(few(criteria, scope, now).map(ids -> slice(ids, offset, limit)).orElseGet(() -> ordered(criteria, scope, now, offset, + limit))); + } + + @Override + public UserPage page(UserCriteria criteria, Specification scope, Instant now, long offset, int limit) + { + Optional> few = few(criteria, scope, now); + if (few.isPresent()) { + return new UserPage(rows(slice(few.get(), offset, limit)), few.get().size()); + } + return new UserPage(rows(ordered(criteria, scope, now, offset, limit)), count(criteria, scope, now)); + } + + /** All matches, newest first, if there are at most {@value #SMALL}; empty if there are more. */ + private Optional> few(UserCriteria criteria, Specification scope, Instant now) + { + CriteriaBuilder builder = entities.getCriteriaBuilder(); + CriteriaQuery query = builder.createTupleQuery(); + Root account = query.from(UserAccount.class); + query.multiselect(account.get("id"), account.get("createdAt")).where(where(criteria, scope, now, account, query, builder)); + List matches = new ArrayList<>(entities.createQuery(query).setMaxResults(SMALL + 1).getResultList()); + if (matches.size() > SMALL) { + return Optional.empty(); + } + matches.sort(NEWEST_FIRST); + return Optional.of(matches.stream().map(match -> match.get(0, Long.class)).toList()); + } + + private static List slice(List ids, long offset, int limit) + { + return offset >= ids.size() ? List.of() : ids.subList(toIntExact(offset), toIntExact(Math.min(ids.size(), offset + limit))); + } + + /** A page of matches in the database's order, along the creation-order index. */ + private List ordered(UserCriteria criteria, Specification scope, Instant now, long offset, int limit) { CriteriaBuilder builder = entities.getCriteriaBuilder(); CriteriaQuery query = builder.createQuery(Long.class); Root account = query.from(UserAccount.class); query.select(account.get("id")).where(where(criteria, scope, now, account, query, builder)) .orderBy(builder.desc(account.get("createdAt")), builder.desc(account.get("id"))); - List ids = entities.createQuery(query).setFirstResult(toIntExact(offset)).setMaxResults(limit).getResultList(); + return entities.createQuery(query).setFirstResult(toIntExact(offset)).setMaxResults(limit).getResultList(); + } + + private List rows(List ids) + { if (ids.isEmpty()) { return List.of(); } diff --git a/core/grantforge-identity/src/main/resources/db/changelog/identity.yaml b/core/grantforge-identity/src/main/resources/db/changelog/identity.yaml index 89060ccf..a7a09e42 100644 --- a/core/grantforge-identity/src/main/resources/db/changelog/identity.yaml +++ b/core/grantforge-identity/src/main/resources/db/changelog/identity.yaml @@ -1056,3 +1056,18 @@ databaseChangeLog: referencedTableName: gf_identity_source referencedColumnNames: id constraintName: fk_gf_external_identity_source + - changeSet: + id: identity-0013-index-gf-user-account-created + author: grantforge + comment: The account list pages newest first; without this index every page sorts the whole tenant (D-86). + changes: + - createIndex: + tableName: gf_user_account + indexName: ix_gf_user_account_created + columns: + - column: + name: tenant_id + - column: + name: created_at + - column: + name: id diff --git a/core/grantforge-identity/src/test/java/org/devlive/grantforge/identity/domain/UserPageTest.java b/core/grantforge-identity/src/test/java/org/devlive/grantforge/identity/domain/UserPageTest.java new file mode 100644 index 00000000..4e6f4da0 --- /dev/null +++ b/core/grantforge-identity/src/test/java/org/devlive/grantforge/identity/domain/UserPageTest.java @@ -0,0 +1,29 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.identity.domain; + +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class UserPageTest +{ + @Test + void keepsACopyOfItsRows() + { + UserRow alice = new UserRow(1, "alice", null, null, AccountStatus.ACTIVE, null, false, false, null, Instant.EPOCH, null, null); + List rows = new ArrayList<>(List.of(alice)); + UserPage page = new UserPage(rows, 7); + rows.clear(); + + assertThat(page.rows()).containsExactly(alice); + assertThat(page.total()).isEqualTo(7); + } +} diff --git a/core/grantforge-identity/src/test/java/org/devlive/grantforge/identity/domain/UserSearchRepositoryTest.java b/core/grantforge-identity/src/test/java/org/devlive/grantforge/identity/domain/UserSearchRepositoryTest.java index b472a5cd..2cd42117 100644 --- a/core/grantforge-identity/src/test/java/org/devlive/grantforge/identity/domain/UserSearchRepositoryTest.java +++ b/core/grantforge-identity/src/test/java/org/devlive/grantforge/identity/domain/UserSearchRepositoryTest.java @@ -19,8 +19,10 @@ import java.time.Duration; import java.time.Instant; +import java.util.Comparator; import java.util.List; import java.util.function.Supplier; +import java.util.stream.IntStream; import static org.assertj.core.api.Assertions.assertThat; @@ -108,6 +110,38 @@ private List names(UserCriteria criteria) return inTenant(() -> accounts.search(criteria, EVERYONE, NOW, 0, 50)).stream().map(UserRow::username).sorted().toList(); } + @Test + void pagesNewestFirstWithTheTotal() + { + List all = inTenant(() -> accounts.search(UserCriteria.ALL, EVERYONE, NOW, 0, 50)); + UserPage first = inTenant(() -> accounts.page(UserCriteria.ALL, EVERYONE, NOW, 0, 2)); + + assertThat(first.total()).isEqualTo(5); + assertThat(first.rows()).containsExactlyElementsOf(all.subList(0, 2)); + assertThat(all).isSortedAccordingTo(Comparator.comparing(UserRow::createdAt).thenComparing(UserRow::id).reversed()); + UserPage last = inTenant(() -> accounts.page(UserCriteria.ALL, EVERYONE, NOW, 4, 2)); + assertThat(last.rows()).containsExactly(all.get(4)); + assertThat(last.total()).isEqualTo(5); + UserPage beyond = inTenant(() -> accounts.page(UserCriteria.ALL, EVERYONE, NOW, 10, 2)); + assertThat(beyond.rows()).isEmpty(); + assertThat(beyond.total()).isEqualTo(5); + assertThat(inTenant(() -> accounts.page(new UserCriteria("ali", null, null, null), EVERYONE, NOW, 0, 2)).total()).isOne(); + } + + @Test + void countsWhenThereAreManyMatches() + { + inTenant(() -> accounts.saveAll(IntStream.range(0, UserSearchRepositoryImpl.SMALL) + .mapToObj(index -> UserAccount.create(String.format("bulk-%04d", index), "h", NOW)).toList())); + List newest = inTenant(() -> accounts.search(UserCriteria.ALL, EVERYONE, NOW, 0, 3)); + + UserPage page = inTenant(() -> accounts.page(UserCriteria.ALL, EVERYONE, NOW, 0, 3)); + assertThat(page.total()).isEqualTo(UserSearchRepositoryImpl.SMALL + 5L); + assertThat(page.rows()).containsExactlyElementsOf(newest).hasSize(3); + assertThat(inTenant(() -> accounts.page(new UserCriteria("bulk-000", null, null, null), EVERYONE, NOW, 0, 20)).total()) + .isEqualTo(10); + } + @Test void listsEveryAccountWithItsPrimaryDepartment() { From d7b642247741e0df6100f1aed6baa5521a4cdc8a Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 01:36:10 -0400 Subject: [PATCH 03/22] perf(identity): index account search with trigrams on PostgreSQL A search for accounts whose name, display name or address contains the text scanned every account; at a million that took 250-400 ms. On PostgreSQL the migration enables pg_trgm where the database allows it and adds trigram indexes matching the search's expressions, which brings the search to a few tens of milliseconds. Without the extension, and on the other databases, nothing changes. --- .../main/resources/db/changelog/identity.yaml | 29 +++++++++++++++++++ docs/content/start/databases.md | 2 ++ 2 files changed, 31 insertions(+) diff --git a/core/grantforge-identity/src/main/resources/db/changelog/identity.yaml b/core/grantforge-identity/src/main/resources/db/changelog/identity.yaml index a7a09e42..25de116d 100644 --- a/core/grantforge-identity/src/main/resources/db/changelog/identity.yaml +++ b/core/grantforge-identity/src/main/resources/db/changelog/identity.yaml @@ -1071,3 +1071,32 @@ databaseChangeLog: name: created_at - column: name: id + - changeSet: + id: identity-0014-enable-pg-trgm + author: grantforge + dbms: postgresql + # Trigram indexes make the account search's "contains" match fast on PostgreSQL (D-87). Trusted from PostgreSQL 13 + # on, so the database owner may enable it; where that is refused the search scans as on the other databases. + failOnError: false + comment: Enables pg_trgm for the account search where the database allows it (D-87) + changes: + - sql: + sql: CREATE EXTENSION IF NOT EXISTS pg_trgm + - changeSet: + id: identity-0015-trigram-gf-user-account + author: grantforge + dbms: postgresql + # Retried at every start until pg_trgm is there, so enabling it later still brings the indexes. + preConditions: + - onFail: CONTINUE + - sqlCheck: + expectedResult: 1 + sql: SELECT COUNT(*) FROM pg_extension WHERE extname = 'pg_trgm' + comment: The account search matches login names, display names and addresses that contain the text (D-87) + changes: + - sql: + sql: CREATE INDEX ix_gf_user_account_uname_trgm ON gf_user_account USING gin (username_norm gin_trgm_ops) + - sql: + sql: CREATE INDEX ix_gf_user_account_dname_trgm ON gf_user_account USING gin (LOWER(display_name) gin_trgm_ops) + - sql: + sql: CREATE INDEX ix_gf_user_account_email_trgm ON gf_user_account USING gin (LOWER(email) gin_trgm_ops) diff --git a/docs/content/start/databases.md b/docs/content/start/databases.md index f9dc7b8a..c6e2d20a 100644 --- a/docs/content/start/databases.md +++ b/docs/content/start/databases.md @@ -39,6 +39,8 @@ GRANTFORGE_DB_URL=jdbc:sqlserver://db:1433;databaseName=grantforge;encrypt=true; 用户名与密码分别用 `GRANTFORGE_DB_USER` 与 `GRANTFORGE_DB_PASSWORD` 设置。数据库需要事先创建，账号需要建表权限：首次启动时 GrantForge 用 Liquibase 创建全部表，之后的版本升级也由 Liquibase 自动迁移。Hibernate 只校验表结构，从不修改它。 +在 PostgreSQL 上，GrantForge 会尝试启用 `pg_trgm` 扩展，并为用户的登录名、显示名与邮箱建立三元组索引，使百万级账号的“包含”搜索保持在几十毫秒。PostgreSQL 13 起它是可信扩展，数据库所有者即可启用；如果账号没有这个权限，服务照常启动，搜索改为全表扫描，由管理员执行 `CREATE EXTENSION pg_trgm` 后下次启动会自动补建索引。 + ## 字符集 - **MySQL / MariaDB**：创建数据库时使用 `utf8mb4` 字符集，中文与表情符号才能完整保存。 From e8b651b7594c5f139b7a2deebf88e7d24ab5f5c8 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 01:45:05 -0400 Subject: [PATCH 04/22] feat(hdfs): add the HDFS service type plugin shipped with the release The hdfs service type declares paths, matched as paths and optionally with everything below them, with read, write and execute. Testing a connection and looking up paths go through WebHDFS or HttpFS with the JDK's HTTP client, so no Hadoop client is needed and any Hadoop version works; several NameNode addresses are tried in turn, skipping standby ones. The plugin is packaged with its own Jackson and the release unpacks it into plugins/hdfs. --- configure/assembly/server.xml | 10 + core/grantforge-plugin-host/pom.xml | 7 + .../plugin/host/HdfsPluginTest.java | 125 +++++++++ core/grantforge-server/pom.xml | 9 + docs/content/guide/data-services.md | 11 + plugins/grantforge-plugin-hdfs/pom.xml | 74 +++++ .../src/assembly/plugin.xml | 39 +++ .../devlive/grantforge/hdfs/HdfsProvider.java | 128 +++++++++ .../org/devlive/grantforge/hdfs/WebHdfs.java | 254 ++++++++++++++++++ .../devlive/grantforge/hdfs/package-info.java | 10 + .../src/main/resources/grantforge-plugin.yaml | 12 + .../devlive/grantforge/hdfs/FakeWebHdfs.java | 105 ++++++++ .../grantforge/hdfs/HdfsProviderTest.java | 103 +++++++ .../devlive/grantforge/hdfs/WebHdfsTest.java | 112 ++++++++ pom.xml | 1 + 15 files changed, 1000 insertions(+) create mode 100644 core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/HdfsPluginTest.java create mode 100644 plugins/grantforge-plugin-hdfs/pom.xml create mode 100644 plugins/grantforge-plugin-hdfs/src/assembly/plugin.xml create mode 100644 plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HdfsProvider.java create mode 100644 plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/WebHdfs.java create mode 100644 plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/package-info.java create mode 100644 plugins/grantforge-plugin-hdfs/src/main/resources/grantforge-plugin.yaml create mode 100644 plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/FakeWebHdfs.java create mode 100644 plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HdfsProviderTest.java create mode 100644 plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/WebHdfsTest.java diff --git a/configure/assembly/server.xml b/configure/assembly/server.xml index b409c5ed..366a42bc 100644 --- a/configure/assembly/server.xml +++ b/configure/assembly/server.xml @@ -20,6 +20,16 @@ lib false + + + false + provided + + org.devlive.grantforge:grantforge-plugin-hdfs:zip:plugin + + plugins/hdfs + true + diff --git a/core/grantforge-plugin-host/pom.xml b/core/grantforge-plugin-host/pom.xml index b8b2261e..2b74f248 100644 --- a/core/grantforge-plugin-host/pom.xml +++ b/core/grantforge-plugin-host/pom.xml @@ -66,6 +66,13 @@ grantforge-plugin-example test + + + org.devlive.grantforge + grantforge-plugin-hdfs + ${project.version} + test + org.devlive.grantforge grantforge-test-support diff --git a/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/HdfsPluginTest.java b/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/HdfsPluginTest.java new file mode 100644 index 00000000..670abbc4 --- /dev/null +++ b/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/HdfsPluginTest.java @@ -0,0 +1,125 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.plugin.host; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.sun.net.httpserver.HttpServer; +import org.devlive.grantforge.hdfs.HdfsProvider; +import org.devlive.grantforge.plugin.api.ConnectionResult; +import org.devlive.grantforge.plugin.api.LookupRequest; +import org.devlive.grantforge.plugin.api.PluginDescriptor; +import org.devlive.grantforge.plugin.api.ServiceConfig; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import tools.jackson.core.JsonParser; +import tools.jackson.databind.json.JsonMapper; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Installs the HDFS plugin as the release ships it, a directory with its classes and its own Jackson in lib/, and talks + * to a WebHDFS endpoint through it: the plugin must work in its own class loader, which sees none of the server's + * libraries. + */ +class HdfsPluginTest +{ + @TempDir + private Path plugins; + + private static Path location(Class type) throws URISyntaxException + { + return Path.of(type.getProtectionDomain().getCodeSource().getLocation().toURI()); + } + + private void install() throws IOException, URISyntaxException + { + Path target = plugins.resolve("hdfs"); + Path built = location(HdfsProvider.class); + Files.createDirectories(target.resolve("lib")); + for (Class library : List.of(JsonMapper.class, JsonParser.class, JsonProperty.class)) { + Path jar = location(library); + Files.copy(jar, target.resolve("lib").resolve(jar.getFileName().toString()), StandardCopyOption.REPLACE_EXISTING); + } + if (Files.isRegularFile(built)) { + Files.copy(built, target.resolve("lib").resolve("grantforge-plugin-hdfs.jar")); + try (var jar = new java.util.jar.JarFile(built.toFile())) { + Files.copy(jar.getInputStream(jar.getEntry(PluginDescriptor.FILE_NAME)), target.resolve(PluginDescriptor.FILE_NAME)); + } + return; + } + Files.copy(built.resolve(PluginDescriptor.FILE_NAME), target.resolve(PluginDescriptor.FILE_NAME)); + try (Stream files = Files.walk(built)) { + for (Path file : files.filter(Files::isRegularFile).toList()) { + Path copy = target.resolve("classes").resolve(built.relativize(file).toString()); + Files.createDirectories(copy.getParent()); + Files.copy(file, copy, StandardCopyOption.REPLACE_EXISTING); + } + } + } + + @Test + void loadsWithItsOwnJacksonAndReachesWebHdfs() throws Exception + { + install(); + HttpServer namenode = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + namenode.createContext("/webhdfs/v1", exchange -> { + String body = exchange.getRequestURI().getQuery().contains("LISTSTATUS") + ? "{\"FileStatuses\":{\"FileStatus\":[{\"pathSuffix\":\"user\",\"type\":\"DIRECTORY\"}]}}" + : "{\"FileStatus\":{\"pathSuffix\":\"\",\"type\":\"DIRECTORY\"}}"; + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, bytes.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(bytes); + } + }); + namenode.start(); + PluginSwitches on = new PluginSwitches() + { + @Override + public boolean enabled(String pluginId) + { + return true; + } + + @Override + public void set(String pluginId, boolean enabled) + { + // Always on. + } + }; + ServiceConfig config = new ServiceConfig("lake", Map.of("url", "http://127.0.0.1:" + namenode.getAddress().getPort())); + try (PluginCalls calls = new PluginCalls(Duration.ofSeconds(10)); + PluginRegistry registry = new PluginRegistry(plugins, on, calls, HdfsPluginTest.class.getClassLoader())) { + registry.scan(); + InstalledPlugin hdfs = registry.plugins().stream().filter(plugin -> plugin.id().equals("hdfs")).findFirst().orElseThrow(); + assertThat(hdfs.status()).as(String.valueOf(hdfs.problem())).isEqualTo(PluginStatus.ACTIVE); + assertThat(registry.serviceType("hdfs")).map(type -> type.label()).contains("HDFS"); + ConnectionResult connected = registry.call("hdfs", provider -> { + assertThat(provider.getClass().getClassLoader()).isInstanceOf(PluginClassLoader.class); + return provider.testConnection(config); + }); + assertThat(connected.status()).as(String.valueOf(connected.message())).isEqualTo(ConnectionResult.Status.SUCCEEDED); + List found = registry.call("hdfs", provider -> provider.lookup(new LookupRequest(config, "path", "/u", Map.of(), 10))); + assertThat(found).containsExactly("/user"); + } + finally { + namenode.stop(0); + } + } +} diff --git a/core/grantforge-server/pom.xml b/core/grantforge-server/pom.xml index b10d9433..42f9ad02 100644 --- a/core/grantforge-server/pom.xml +++ b/core/grantforge-server/pom.xml @@ -65,6 +65,15 @@ org.springframework.boot spring-boot-starter-liquibase + + + org.devlive.grantforge + grantforge-plugin-hdfs + ${project.version} + plugin + zip + provided + org.springframework.boot diff --git a/docs/content/guide/data-services.md b/docs/content/guide/data-services.md index de17b0cf..949dea83 100644 --- a/docs/content/guide/data-services.md +++ b/docs/content/guide/data-services.md @@ -28,6 +28,17 @@ flowchart LR ![插件](/screenshots/plugins.png) +## HDFS + +发行包自带 HDFS 插件（`plugins/hdfs`），服务类型 `hdfs`： + +- 资源只有一级 `path`，按路径匹配：`/data/sales` 匹配它本身，勾选“递归”后也匹配其下的全部文件与目录；支持排除。 +- 访问类型 `read`、`write`、`execute`，与 HDFS 的权限位对应。 +- 配置：WebHDFS 地址（NameNode 的 HTTP 地址，如 `http://namenode:9870`；高可用时用逗号分隔两个 NameNode，自动找到 active 的那个；也可以填 HttpFS 地址）、查询目录用的用户（默认 `hdfs`，简单认证）与超时时间。 +- “测试连接”读取根目录的状态；写策略时输入路径会列出对应目录下的子目录与文件供选择。 + +插件通过 WebHDFS 的 REST 接口访问 HDFS，不依赖 Hadoop 客户端，适用于各个 Hadoop 版本。开启 Kerberos 的集群暂不支持测试连接与路径补全，策略仍可手工填写。 + ## 数据服务 **数据权限 → 数据服务**：一个服务是 GrantForge 管理权限的一个外部系统实例，例如一个 HDFS 集群。添加服务时选择服务类型，按插件定义的配置项填写连接信息，可以先 **测试连接**。密码等敏感配置加密保存，保存后不再显示。 diff --git a/plugins/grantforge-plugin-hdfs/pom.xml b/plugins/grantforge-plugin-hdfs/pom.xml new file mode 100644 index 00000000..3462d8bc --- /dev/null +++ b/plugins/grantforge-plugin-hdfs/pom.xml @@ -0,0 +1,74 @@ + + + + + + grantforge + org.devlive.grantforge + 2026.0.0 + ../../pom.xml + + 4.0.0 + + grantforge-plugin-hdfs + GrantForge HDFS Plugin + + The HDFS service type (M13-03): paths with read, write and execute, looked up and tested through WebHDFS or + HttpFS over HTTP, so it works with every Hadoop version without the Hadoop client. Packaged as a plugin + directory (descriptor, classes and lib/) that the release ships in plugins/hdfs. + + + + + org.devlive.grantforge + grantforge-plugin-api + provided + + + + tools.jackson.core + jackson-databind + + + org.jspecify + jspecify + provided + + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + src/assembly/plugin.xml + + + + + plugin-package + package + + single + + + + + + + diff --git a/plugins/grantforge-plugin-hdfs/src/assembly/plugin.xml b/plugins/grantforge-plugin-hdfs/src/assembly/plugin.xml new file mode 100644 index 00000000..91d2737a --- /dev/null +++ b/plugins/grantforge-plugin-hdfs/src/assembly/plugin.xml @@ -0,0 +1,39 @@ + + + + + + plugin + + zip + + false + + + false + lib + runtime + + + + + ${project.build.outputDirectory} + classes + + grantforge-plugin.yaml + + + + ${project.build.outputDirectory} + / + + grantforge-plugin.yaml + + + + diff --git a/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HdfsProvider.java b/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HdfsProvider.java new file mode 100644 index 00000000..c2921748 --- /dev/null +++ b/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HdfsProvider.java @@ -0,0 +1,128 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.hdfs; + +import org.devlive.grantforge.plugin.api.ConnectionResult; +import org.devlive.grantforge.plugin.api.LookupRequest; +import org.devlive.grantforge.plugin.api.ServiceConfig; +import org.devlive.grantforge.plugin.api.ServiceTypeProvider; +import org.devlive.grantforge.plugin.api.model.AccessTypeDefinition; +import org.devlive.grantforge.plugin.api.model.ConfigField; +import org.devlive.grantforge.plugin.api.model.ConfigFieldType; +import org.devlive.grantforge.plugin.api.model.ConfigProblem; +import org.devlive.grantforge.plugin.api.model.MatcherType; +import org.devlive.grantforge.plugin.api.model.ResourceDefinition; +import org.devlive.grantforge.plugin.api.model.ServiceTypeDefinition; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +/** + * The HDFS service type: paths, matched as paths and optionally with everything below them, with the access types the + * HDFS agent checks, read, write and execute. Connections and path lookups go through WebHDFS or HttpFS. + */ +public final class HdfsProvider + implements ServiceTypeProvider +{ + /** The service type's name. */ + public static final String TYPE = "hdfs"; + + static final String PATH = "path"; + static final String URL = "url"; + static final String USER = "username"; + static final String TIMEOUT = "timeout"; + static final long MAX_TIMEOUT = 120; + + @Override + public ServiceTypeDefinition definition() + { + return ServiceTypeDefinition.builder(TYPE).label("HDFS").description("Paths of the Hadoop Distributed File System") + .resources(ResourceDefinition.builder(PATH).label("Path").matcher(MatcherType.PATH).caseSensitive(true) + .recursiveSupported(true).excludesSupported(true).lookupSupported(true).validLeaf(true).build()) + .accessTypes(AccessTypeDefinition.of("read", "Read"), AccessTypeDefinition.of("write", "Write"), + AccessTypeDefinition.of("execute", "Execute")) + .configFields(ConfigField.builder(URL).label("WebHDFS address").type(ConfigFieldType.STRING).mandatory() + .pattern("https?://\\S+(\\s*,\\s*https?://\\S+)*") + .description("The NameNodes' HTTP addresses, comma-separated for high availability, or HttpFS;" + + " such as http://namenode:9870").build(), + ConfigField.builder(USER).label("User").type(ConfigFieldType.STRING).defaultValue("hdfs") + .description("Who lists paths, with simple authentication").build(), + ConfigField.builder(TIMEOUT).label("Timeout (seconds)").type(ConfigFieldType.INTEGER).defaultValue("10").build()) + .build(); + } + + @Override + public List validateConfig(ServiceConfig config) + { + List problems = new ArrayList<>(); + String url = config.get(URL); + if (url != null) { + try { + WebHdfs.addresses(url); + } + catch (IllegalArgumentException invalid) { + problems.add(new ConfigProblem(URL, ConfigProblem.Reason.INVALID, String.valueOf(invalid.getMessage()))); + } + } + long timeout = config.getLong(TIMEOUT, 10); + if (timeout < 1 || timeout > MAX_TIMEOUT) { + problems.add(new ConfigProblem(TIMEOUT, ConfigProblem.Reason.INVALID, "1 to " + MAX_TIMEOUT + " seconds")); + } + return problems; + } + + @Override + public ConnectionResult testConnection(ServiceConfig config) + { + try { + WebHdfs.Entry root = client(config).status("/"); + return root != null && root.directory() ? ConnectionResult.succeeded() + : ConnectionResult.failed("the root of the file system is not a directory"); + } + catch (IOException | IllegalArgumentException failed) { + return ConnectionResult.failed(String.valueOf(failed.getMessage())); + } + } + + /** + * Lists the paths that start with what was typed: the entries of the directory typed so far whose names start with + * the rest, directories first. + */ + @Override + public List lookup(LookupRequest request) + { + if (!PATH.equals(request.resource())) { + return List.of(); + } + String typed = request.userInput().strip(); + String path = typed.startsWith("/") ? typed : "/" + typed; + int slash = path.lastIndexOf('/'); + String directory = slash == 0 ? "/" : path.substring(0, slash); + String prefix = path.substring(slash + 1); + String base = "/".equals(directory) ? "" : directory; + List entries; + try { + entries = client(request.config()).list(directory); + } + catch (IOException failed) { + throw new UncheckedIOException(failed); + } + return entries.stream().filter(entry -> entry.name().startsWith(prefix)) + .sorted((left, right) -> left.directory() == right.directory() ? left.name().compareTo(right.name()) + : left.directory() ? -1 : 1) + .limit(request.limit()).map(entry -> base + "/" + entry.name()).toList(); + } + + private static WebHdfs client(ServiceConfig config) + { + String user = config.get(USER); + return new WebHdfs(WebHdfs.addresses(config.require(URL)), user == null || user.isBlank() ? "hdfs" : user.strip(), + Duration.ofSeconds(Math.max(1, Math.min(MAX_TIMEOUT, config.getLong(TIMEOUT, 10))))); + } +} diff --git a/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/WebHdfs.java b/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/WebHdfs.java new file mode 100644 index 00000000..af2a4ac1 --- /dev/null +++ b/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/WebHdfs.java @@ -0,0 +1,254 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.hdfs; + +import org.jspecify.annotations.Nullable; +import tools.jackson.core.JacksonException; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.json.JsonMapper; + +import java.io.IOException; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import static java.util.Objects.requireNonNull; + +/** + * The few WebHDFS (or HttpFS) calls the service type needs, over the JDK's HTTP client with simple authentication + * ({@code user.name}). With several NameNode addresses, as in a high-availability pair, the calls go to the first one + * that answers as active: a standby NameNode refuses with a {@code StandbyException} and the next is asked. + */ +final class WebHdfs +{ + private static final JsonMapper JSON = JsonMapper.builder().build(); + private static final String PREFIX = "/webhdfs/v1"; + + private final List addresses; + private final String user; + private final Duration timeout; + private final HttpClient http; + + /** + * Creates the client. + * + * @param addresses the NameNode or HttpFS addresses, such as {@code http://namenode:9870} + * @param user the user to act as + * @param timeout how long a call may take + */ + WebHdfs(List addresses, String user, Duration timeout) + { + if (addresses.isEmpty()) { + throw new IllegalArgumentException("no WebHDFS address"); + } + this.addresses = List.copyOf(addresses); + this.user = requireNonNull(user, "user"); + this.timeout = requireNonNull(timeout, "timeout"); + this.http = HttpClient.newBuilder().connectTimeout(timeout).followRedirects(HttpClient.Redirect.NEVER).build(); + } + + /** + * Parses a comma-separated list of addresses. + * + * @param text the addresses + * @return the addresses, without their trailing slashes + * @throws IllegalArgumentException if one is not an http or https address + */ + static List addresses(String text) + { + List addresses = new ArrayList<>(); + for (String part : text.split(",")) { + String address = part.strip(); + if (address.isEmpty()) { + continue; + } + while (address.endsWith("/")) { + address = address.substring(0, address.length() - 1); + } + URI uri; + try { + uri = URI.create(address); + } + catch (IllegalArgumentException broken) { + throw new IllegalArgumentException(part.strip() + " is not an address", broken); + } + if (!("http".equals(uri.getScheme()) || "https".equals(uri.getScheme())) || uri.getHost() == null) { + throw new IllegalArgumentException(part.strip() + " is not an http or https address"); + } + addresses.add(uri); + } + if (addresses.isEmpty()) { + throw new IllegalArgumentException("no address given"); + } + return addresses; + } + + /** + * Returns the status of a path. + * + * @param path an absolute path + * @return the status, or {@code null} if the path does not exist + * @throws IOException if no NameNode answers, or one refuses + */ + @Nullable Entry status(String path) throws IOException + { + JsonNode answer = call(path, "GETFILESTATUS"); + return answer == null ? null : entry(answer.path("FileStatus")); + } + + /** + * Lists a directory. + * + * @param path an absolute path + * @return its entries, or an empty list if it does not exist + * @throws IOException if no NameNode answers, or one refuses + */ + List list(String path) throws IOException + { + JsonNode answer = call(path, "LISTSTATUS"); + if (answer == null) { + return List.of(); + } + List entries = new ArrayList<>(); + for (JsonNode status : answer.path("FileStatuses").path("FileStatus")) { + entries.add(entry(status)); + } + return entries; + } + + private static Entry entry(JsonNode status) + { + return new Entry(status.path("pathSuffix").asString(""), "DIRECTORY".equals(status.path("type").asString(""))); + } + + /** Calls the first NameNode that answers as active; {@code null} when the path does not exist. */ + private @Nullable JsonNode call(String path, String operation) throws IOException + { + IOException last = null; + for (URI address : addresses) { + try { + return call(address, path, operation); + } + catch (StandbyException standby) { + last = standby; + } + catch (Refused refused) { + throw refused; + } + catch (IOException unreachable) { + last = unreachable; + } + } + throw requireNonNull(last, "no address was asked"); + } + + // Restores the caller's interrupt when the call is interrupted. + @SuppressWarnings("PMD.DoNotUseThreads") + private @Nullable JsonNode call(URI address, String path, String operation) throws IOException + { + URI uri = URI.create(address + PREFIX + encode(path) + "?op=" + operation + "&user.name=" + + URLEncoder.encode(user, StandardCharsets.UTF_8)); + HttpRequest request = HttpRequest.newBuilder(uri).timeout(timeout).header("Accept", "application/json").GET().build(); + HttpResponse response; + try { + response = http.send(request, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); + } + catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IOException("interrupted while asking " + address, interrupted); + } + catch (IOException unreachable) { + throw new IOException(address + " cannot be reached: " + unreachable.getMessage(), unreachable); + } + JsonNode body = json(response.body()); + if (response.statusCode() == 200) { + if (body == null) { + throw new IOException(address + " did not answer in JSON; is it a WebHDFS address?"); + } + return body; + } + JsonNode remote = body == null ? null : body.path("RemoteException"); + String exception = remote == null ? "" : remote.path("exception").asString(""); + String message = remote == null ? "" : remote.path("message").asString(""); + if ("StandbyException".equals(exception)) { + throw new StandbyException(address + " is a standby NameNode"); + } + if (response.statusCode() == 404 && ("FileNotFoundException".equals(exception) || exception.isEmpty())) { + if (remote == null || remote.isMissingNode()) { + throw new Refused(address + " answered 404; is it a WebHDFS address?"); + } + return null; + } + throw new Refused(address + " refused " + operation.toLowerCase(Locale.ROOT) + " " + path + " (" + response.statusCode() + + (exception.isEmpty() ? "" : " " + exception) + (message.isEmpty() ? "" : ": " + message) + ")"); + } + + private static @Nullable JsonNode json(String body) + { + try { + JsonNode node = JSON.readTree(body); + return node != null && node.isObject() ? node : null; + } + catch (JacksonException notJson) { + return null; + } + } + + /** Encodes each segment of a path for the URL, keeping the slashes. */ + static String encode(String path) + { + StringBuilder encoded = new StringBuilder(); + for (String segment : path.split("/", -1)) { + if (encoded.length() > 0 || !segment.isEmpty()) { + encoded.append('/'); + } + encoded.append(URLEncoder.encode(segment, StandardCharsets.UTF_8).replace("+", "%20")); + } + String result = encoded.toString(); + return result.startsWith("/") ? result : "/" + result; + } + + /** + * A directory entry. + * + * @param name its name within the directory + * @param directory whether it is a directory + */ + record Entry(String name, boolean directory) + { + } + + /** A NameNode that is standby, so the next one is asked. */ + static final class StandbyException + extends IOException + { + private static final long serialVersionUID = 1L; + + StandbyException(String message) + { + super(message); + } + } + + /** A NameNode that answered and refused, so asking another does not help. */ + static final class Refused + extends IOException + { + private static final long serialVersionUID = 1L; + + Refused(String message) + { + super(message); + } + } +} diff --git a/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/package-info.java b/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/package-info.java new file mode 100644 index 00000000..0a2916fa --- /dev/null +++ b/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/package-info.java @@ -0,0 +1,10 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +/** The HDFS service type plugin (M13-03): paths with read, write and execute, through WebHDFS. */ +@NullMarked +package org.devlive.grantforge.hdfs; + +import org.jspecify.annotations.NullMarked; diff --git a/plugins/grantforge-plugin-hdfs/src/main/resources/grantforge-plugin.yaml b/plugins/grantforge-plugin-hdfs/src/main/resources/grantforge-plugin.yaml new file mode 100644 index 00000000..77f83eec --- /dev/null +++ b/plugins/grantforge-plugin-hdfs/src/main/resources/grantforge-plugin.yaml @@ -0,0 +1,12 @@ +# Copyright (c) 2026 devlive-community/grantforge +# +# Licensed under the MIT License. See the LICENSE file in the +# project root for full license text. + +id: hdfs +version: 1.0.0 +name: HDFS +description: Paths of the Hadoop Distributed File System with read, write and execute +apiVersion: "1.0" +providers: + - org.devlive.grantforge.hdfs.HdfsProvider diff --git a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/FakeWebHdfs.java b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/FakeWebHdfs.java new file mode 100644 index 00000000..621669e3 --- /dev/null +++ b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/FakeWebHdfs.java @@ -0,0 +1,105 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.hdfs; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * A NameNode's WebHDFS with a small file system: / holds user/ and tmp/, /user holds alice/, bob/ and a file notes.txt. + * As a standby it refuses every call the way Hadoop does. + */ +final class FakeWebHdfs + implements AutoCloseable +{ + private static final Map> TREE = Map.of( + "/", List.of(new String[] {"user", "DIRECTORY"}, new String[] {"tmp", "DIRECTORY"}), + "/user", List.of(new String[] {"bob", "DIRECTORY"}, new String[] {"notes.txt", "FILE"}, new String[] {"alice", "DIRECTORY"}), + "/user/alice", List.of(), + "/user/bob", List.of(), + "/tmp", List.of()); + + final List requests = new CopyOnWriteArrayList<>(); + + private final HttpServer server; + private final boolean standby; + + FakeWebHdfs(boolean standby) throws IOException + { + this.standby = standby; + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/webhdfs/v1", this::handle); + server.createContext("/", exchange -> respond(exchange, 404, "Not Found")); + server.start(); + } + + URI uri() + { + return URI.create("http://127.0.0.1:" + server.getAddress().getPort()); + } + + private void handle(HttpExchange exchange) throws IOException + { + String path = URLDecoder.decode(exchange.getRequestURI().getRawPath().substring("/webhdfs/v1".length()), StandardCharsets.UTF_8); + String query = exchange.getRequestURI().getRawQuery(); + requests.add(path + "?" + query); + if (standby) { + respond(exchange, 403, "{\"RemoteException\":{\"exception\":\"StandbyException\",\"javaClassName\":" + + "\"org.apache.hadoop.ipc.StandbyException\",\"message\":\"Operation category READ is not supported in state standby\"}}"); + return; + } + if (query.contains("user.name=nobody")) { + respond(exchange, 403, "{\"RemoteException\":{\"exception\":\"AccessControlException\",\"message\":\"Permission denied: user=nobody\"}}"); + return; + } + String normal = path.length() > 1 && path.endsWith("/") ? path.substring(0, path.length() - 1) : path; + if (normal.isEmpty()) { + normal = "/"; + } + boolean directory = TREE.containsKey(normal); + boolean file = "/user/notes.txt".equals(normal); + if (!directory && !file) { + respond(exchange, 404, "{\"RemoteException\":{\"exception\":\"FileNotFoundException\",\"message\":\"File does not exist: " + + normal + "\"}}"); + return; + } + if (query.contains("op=GETFILESTATUS")) { + respond(exchange, 200, "{\"FileStatus\":{\"pathSuffix\":\"\",\"type\":\"" + (directory ? "DIRECTORY" : "FILE") + "\"}}"); + return; + } + StringBuilder statuses = new StringBuilder(); + for (String[] entry : TREE.getOrDefault(normal, List.of())) { + statuses.append(statuses.length() == 0 ? "" : ",").append("{\"pathSuffix\":\"").append(entry[0]).append("\",\"type\":\"") + .append(entry[1]).append("\"}"); + } + respond(exchange, 200, "{\"FileStatuses\":{\"FileStatus\":[" + statuses + "]}}"); + } + + private static void respond(HttpExchange exchange, int status, String body) throws IOException + { + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(status, bytes.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(bytes); + } + } + + @Override + public void close() + { + server.stop(0); + } +} diff --git a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HdfsProviderTest.java b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HdfsProviderTest.java new file mode 100644 index 00000000..8b3b6a7f --- /dev/null +++ b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HdfsProviderTest.java @@ -0,0 +1,103 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.hdfs; + +import org.devlive.grantforge.plugin.api.ConnectionResult; +import org.devlive.grantforge.plugin.api.LookupRequest; +import org.devlive.grantforge.plugin.api.ServiceConfig; +import org.devlive.grantforge.plugin.api.model.ConfigProblem; +import org.devlive.grantforge.plugin.api.model.MatcherType; +import org.devlive.grantforge.plugin.api.model.ResourceDefinition; +import org.devlive.grantforge.plugin.api.model.ServiceTypeDefinition; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class HdfsProviderTest +{ + private final HdfsProvider provider = new HdfsProvider(); + + private static ServiceConfig config(String url, String... more) + { + Map values = new HashMap<>(); + values.put(HdfsProvider.URL, url); + for (int index = 0; index < more.length; index += 2) { + values.put(more[index], more[index + 1]); + } + return new ServiceConfig("warehouse-hdfs", values); + } + + private List lookup(ServiceConfig config, String typed) + { + return provider.lookup(new LookupRequest(config, HdfsProvider.PATH, typed, Map.of(), 10)); + } + + @Test + void declaresPathsWithReadWriteAndExecute() + { + ServiceTypeDefinition definition = provider.definition(); + + assertThat(definition.name()).isEqualTo("hdfs"); + ResourceDefinition path = definition.resources().get(0); + assertThat(path.name()).isEqualTo("path"); + assertThat(path.matcher()).isEqualTo(MatcherType.PATH); + assertThat(path.recursiveSupported()).isTrue(); + assertThat(path.lookupSupported()).isTrue(); + assertThat(path.caseSensitive()).isTrue(); + assertThat(definition.accessTypes()).extracting(access -> access.name()).containsExactly("read", "write", "execute"); + assertThat(definition.configFields()).extracting(field -> field.name()).containsExactly("url", "username", "timeout"); + } + + @Test + void checksAddressesAndTimeouts() + { + assertThat(provider.validateConfig(config("http://nn1:9870,http://nn2:9870", "timeout", "30"))).isEmpty(); + assertThat(provider.validateConfig(config("ftp://nn1"))).extracting(ConfigProblem::field).containsExactly("url"); + assertThat(provider.validateConfig(config("http://nn1:9870", "timeout", "0"))).extracting(ConfigProblem::field) + .containsExactly("timeout"); + assertThat(provider.validateConfig(config("http://nn1:9870", "timeout", "121"))).extracting(ConfigProblem::reason) + .containsExactly(ConfigProblem.Reason.INVALID); + assertThat(provider.validateConfig(new ServiceConfig("warehouse-hdfs", Map.of()))).isEmpty(); + } + + @Test + void testsTheConnectionOnTheRoot() throws IOException + { + try (FakeWebHdfs namenode = new FakeWebHdfs(false)) { + assertThat(provider.testConnection(config(namenode.uri().toString()))).isEqualTo(ConnectionResult.succeeded()); + ConnectionResult refused = provider.testConnection(config(namenode.uri().toString(), "username", "nobody")); + assertThat(refused.status()).isEqualTo(ConnectionResult.Status.FAILED); + assertThat(refused.message()).contains("Permission denied"); + } + assertThat(provider.testConnection(config("http://127.0.0.1:1")).message()).contains("cannot be reached"); + assertThat(provider.testConnection(config("not an address")).status()).isEqualTo(ConnectionResult.Status.FAILED); + } + + @Test + void looksUpPathsBelowWhatWasTyped() throws IOException + { + try (FakeWebHdfs namenode = new FakeWebHdfs(false)) { + ServiceConfig config = config(namenode.uri().toString(), "username", " "); + + assertThat(lookup(config, "")).containsExactly("/tmp", "/user"); + assertThat(lookup(config, "/user/")).containsExactly("/user/alice", "/user/bob", "/user/notes.txt"); + assertThat(lookup(config, "user/a")).containsExactly("/user/alice"); + assertThat(lookup(config, "/user/n")).containsExactly("/user/notes.txt"); + assertThat(lookup(config, "/missing/x")).isEmpty(); + assertThat(provider.lookup(new LookupRequest(config, "other", "", Map.of(), 10))).isEmpty(); + assertThat(provider.lookup(new LookupRequest(config, HdfsProvider.PATH, "/user/", Map.of(), 1))).hasSize(1); + assertThat(namenode.requests).allMatch(request -> request.contains("user.name=hdfs")); + } + assertThatThrownBy(() -> lookup(config("http://127.0.0.1:1"), "/")).isInstanceOf(UncheckedIOException.class); + } +} diff --git a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/WebHdfsTest.java b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/WebHdfsTest.java new file mode 100644 index 00000000..84318e6e --- /dev/null +++ b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/WebHdfsTest.java @@ -0,0 +1,112 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.hdfs; + +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class WebHdfsTest +{ + private static final Duration TIMEOUT = Duration.ofSeconds(5); + + @Test + void parsesCommaSeparatedAddresses() + { + assertThat(WebHdfs.addresses(" http://nn1:9870/ , https://nn2:9871,")) + .containsExactly(URI.create("http://nn1:9870"), URI.create("https://nn2:9871")); + assertThatIllegalArgumentException().isThrownBy(() -> WebHdfs.addresses(" , ")).withMessageContaining("no address"); + assertThatIllegalArgumentException().isThrownBy(() -> WebHdfs.addresses("hdfs://nn1:8020")).withMessageContaining("http"); + assertThatIllegalArgumentException().isThrownBy(() -> WebHdfs.addresses("http://")).withMessageContaining("http"); + assertThatIllegalArgumentException().isThrownBy(() -> WebHdfs.addresses("http://bad host")).withMessageContaining("not an address"); + assertThatIllegalArgumentException().isThrownBy(() -> new WebHdfs(List.of(), "hdfs", TIMEOUT)).withMessageContaining("no WebHDFS"); + } + + @Test + void encodesPathSegments() + { + assertThat(WebHdfs.encode("/")).isEqualTo("/"); + assertThat(WebHdfs.encode("/user/a b/x+y")).isEqualTo("/user/a%20b/x%2By"); + assertThat(WebHdfs.encode("data/")).isEqualTo("/data/"); + } + + @Test + void readsStatusesAndListingsAsTheUser() throws IOException + { + try (FakeWebHdfs namenode = new FakeWebHdfs(false)) { + WebHdfs client = new WebHdfs(List.of(namenode.uri()), "alice", TIMEOUT); + + assertThat(client.status("/")).isEqualTo(new WebHdfs.Entry("", true)); + assertThat(client.status("/user/notes.txt")).isEqualTo(new WebHdfs.Entry("", false)); + assertThat(client.status("/missing")).isNull(); + assertThat(client.list("/user")).contains(new WebHdfs.Entry("alice", true), new WebHdfs.Entry("notes.txt", false)); + assertThat(client.list("/missing")).isEmpty(); + assertThat(namenode.requests).allMatch(request -> request.contains("user.name=alice")); + } + } + + @Test + void asksTheActiveNameNodeOfAPair() throws IOException + { + try (FakeWebHdfs standby = new FakeWebHdfs(true); FakeWebHdfs active = new FakeWebHdfs(false)) { + WebHdfs client = new WebHdfs(List.of(standby.uri(), active.uri()), "hdfs", TIMEOUT); + + assertThat(client.list("/")).hasSize(2); + assertThat(standby.requests).hasSize(1); + WebHdfs onlyStandby = new WebHdfs(List.of(standby.uri()), "hdfs", TIMEOUT); + assertThatThrownBy(() -> onlyStandby.list("/")).isInstanceOf(WebHdfs.StandbyException.class).hasMessageContaining("standby"); + } + } + + @Test + void reportsRefusalsAndUnreachableNameNodes() throws IOException + { + try (FakeWebHdfs namenode = new FakeWebHdfs(false)) { + WebHdfs nobody = new WebHdfs(List.of(namenode.uri()), "nobody", TIMEOUT); + assertThatThrownBy(() -> nobody.list("/")).isInstanceOf(WebHdfs.Refused.class).hasMessageContaining("403") + .hasMessageContaining("AccessControlException").hasMessageContaining("Permission denied"); + // A refusal is final: the next address is not asked. + WebHdfs pair = new WebHdfs(List.of(namenode.uri(), URI.create("http://127.0.0.1:1")), "nobody", TIMEOUT); + assertThatThrownBy(() -> pair.list("/")).isInstanceOf(WebHdfs.Refused.class); + } + WebHdfs gone = new WebHdfs(List.of(URI.create("http://127.0.0.1:1")), "hdfs", Duration.ofSeconds(2)); + assertThatThrownBy(() -> gone.list("/")).isInstanceOf(IOException.class).hasMessageContaining("cannot be reached"); + } + + @Test + void tellsWhenTheAddressIsNotWebHdfs() throws IOException + { + HttpServer web = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + web.createContext("/", exchange -> { + boolean ok = exchange.getRequestURI().getQuery().contains("GETFILESTATUS"); + byte[] body = "hello".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(ok ? 200 : 404, body.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(body); + } + }); + web.start(); + try { + WebHdfs client = new WebHdfs(List.of(URI.create("http://127.0.0.1:" + web.getAddress().getPort())), "hdfs", TIMEOUT); + assertThatThrownBy(() -> client.status("/")).isInstanceOf(IOException.class).hasMessageContaining("JSON"); + assertThatThrownBy(() -> client.list("/")).isInstanceOf(WebHdfs.Refused.class).hasMessageContaining("WebHDFS address"); + } + finally { + web.stop(0); + } + } +} diff --git a/pom.xml b/pom.xml index 99ac6a3d..9b815a16 100644 --- a/pom.xml +++ b/pom.xml @@ -34,6 +34,7 @@ core/grantforge-server sdk/grantforge-spring-boot-starter plugins/grantforge-plugin-example + plugins/grantforge-plugin-hdfs From b7c3c9af92a993f0f0364b609de8dfc0c5b3a2ea Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 02:09:54 -0400 Subject: [PATCH 05/22] fix(identity): page accounts without Optional.get The architecture rules forbid Optional.get, which the account page used, so every test job failed on the identity module's architecture test. --- .../identity/domain/UserSearchRepositoryImpl.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/domain/UserSearchRepositoryImpl.java b/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/domain/UserSearchRepositoryImpl.java index 033e016e..f58f620f 100644 --- a/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/domain/UserSearchRepositoryImpl.java +++ b/core/grantforge-identity/src/main/java/org/devlive/grantforge/identity/domain/UserSearchRepositoryImpl.java @@ -73,11 +73,8 @@ public List search(UserCriteria criteria, Specification sc @Override public UserPage page(UserCriteria criteria, Specification scope, Instant now, long offset, int limit) { - Optional> few = few(criteria, scope, now); - if (few.isPresent()) { - return new UserPage(rows(slice(few.get(), offset, limit)), few.get().size()); - } - return new UserPage(rows(ordered(criteria, scope, now, offset, limit)), count(criteria, scope, now)); + return few(criteria, scope, now).map(ids -> new UserPage(rows(slice(ids, offset, limit)), ids.size())) + .orElseGet(() -> new UserPage(rows(ordered(criteria, scope, now, offset, limit)), count(criteria, scope, now))); } /** All matches, newest first, if there are at most {@value #SMALL}; empty if there are more. */ From e84955932c4dcb299d42ddf8a0c6a7559504325d Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 02:09:54 -0400 Subject: [PATCH 06/22] perf(authz): work out snapshots in read-only transactions Every API call reads the console snapshot; the evaluator only reads, so its transactions skip flushing and dirty checking. The cached path's p99 was 1.014 ms against a 1 ms limit. --- .../grantforge/authz/application/AuthorizationEvaluator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/grantforge-authz/src/main/java/org/devlive/grantforge/authz/application/AuthorizationEvaluator.java b/core/grantforge-authz/src/main/java/org/devlive/grantforge/authz/application/AuthorizationEvaluator.java index 76d15fc9..16c53851 100644 --- a/core/grantforge-authz/src/main/java/org/devlive/grantforge/authz/application/AuthorizationEvaluator.java +++ b/core/grantforge-authz/src/main/java/org/devlive/grantforge/authz/application/AuthorizationEvaluator.java @@ -107,6 +107,8 @@ public AuthorizationEvaluator(EffectiveRoles effectiveRoles, RoleGrantRepository this.dependencies = requireNonNull(dependencies, "dependencies"); this.applications = requireNonNull(applications, "applications"); this.transactions = new TransactionTemplate(requireNonNull(transactionManager, "transactionManager")); + // Snapshots only read: no flush or dirty checking on the path every API call takes (D-86). + this.transactions.setReadOnly(true); this.clock = requireNonNull(clock, "clock"); } From 4359141852c67423ae7491e45d987f9e9b77d73f Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 02:09:54 -0400 Subject: [PATCH 07/22] feat(plugin-host): load the repository's plugins when run from sources A server started from an IDE runs from build output folders and has no plugins folder, so no plugin loaded. Without an explicit grantforge.plugins.directory and without a plugins folder in the working directory, the host now uses the plugins folder of the repository its classes were built in, and a built plugin module loads from target/classes with the dependencies its build copied to target/plugin-lib. Server and service tests keep an empty plugins folder. --- .../plugin/host/PluginDirectory.java | 90 +++++++++++++++++++ .../plugin/host/PluginHostConfiguration.java | 14 +-- .../grantforge/plugin/host/PluginPackage.java | 42 ++++++++- .../plugin/host/PluginDirectoryTest.java | 58 ++++++++++++ .../host/PluginHostConfigurationTest.java | 2 +- .../plugin/host/PluginPackageTest.java | 32 +++++++ .../resources/config/application.properties | 8 ++ .../resources/config/application.properties | 8 ++ docs/content/architecture/development.md | 4 + docs/content/architecture/plugins.md | 27 ++++++ 10 files changed, 275 insertions(+), 10 deletions(-) create mode 100644 core/grantforge-plugin-host/src/main/java/org/devlive/grantforge/plugin/host/PluginDirectory.java create mode 100644 core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/PluginDirectoryTest.java create mode 100644 core/grantforge-server/src/test/resources/config/application.properties create mode 100644 core/grantforge-service/src/test/resources/config/application.properties diff --git a/core/grantforge-plugin-host/src/main/java/org/devlive/grantforge/plugin/host/PluginDirectory.java b/core/grantforge-plugin-host/src/main/java/org/devlive/grantforge/plugin/host/PluginDirectory.java new file mode 100644 index 00000000..f400699a --- /dev/null +++ b/core/grantforge-plugin-host/src/main/java/org/devlive/grantforge/plugin/host/PluginDirectory.java @@ -0,0 +1,90 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.plugin.host; + +import org.jspecify.annotations.Nullable; + +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.CodeSource; + +/** + * Picks the plugins directory (D-89). An explicit {@code grantforge.plugins.directory} wins; otherwise + * {@code plugins} in the working directory, as a release has it. A server started from the sources, as an IDE starts + * it from wherever, has no such folder: when its classes come from a build's output folder, the {@code plugins} folder + * of the repository they were built in is used, whose built plugin modules then load as plugins. + */ +final class PluginDirectory +{ + /** The folder a release keeps its plugins in, relative to the working directory. */ + static final String STANDARD = "plugins"; + + private PluginDirectory() + { + } + + /** + * Picks the directory. + * + * @param configured the setting; blank when not set + * @param codeSource where the server's classes come from, or {@code null} if unknown + * @return the directory to scan + */ + static Path choose(String configured, @Nullable Path codeSource) + { + if (!configured.isBlank()) { + return Path.of(configured.strip()); + } + Path standard = Path.of(STANDARD); + if (Files.isDirectory(standard)) { + return standard; + } + Path sources = sourceTree(codeSource); + return sources == null ? standard : sources; + } + + /** + * Finds the {@code plugins} folder of the repository classes were built in. + * + * @param codeSource a build output folder, such as {@code core/grantforge-plugin-host/target/classes}; a jar is a + * release, which has no repository + * @return the folder, or {@code null} + */ + static @Nullable Path sourceTree(@Nullable Path codeSource) + { + if (codeSource == null || !Files.isDirectory(codeSource)) { + return null; + } + for (Path directory = codeSource.toAbsolutePath(); directory != null; directory = directory.getParent()) { + if (Files.isRegularFile(directory.resolve("pom.xml")) && Files.isDirectory(directory.resolve(STANDARD)) + && Files.isDirectory(directory.resolve("core"))) { + return directory.resolve(STANDARD); + } + } + return null; + } + + /** + * Where a class was loaded from. + * + * @param type the class + * @return its jar or class folder, or {@code null} if that is not a file + */ + static @Nullable Path codeSource(Class type) + { + CodeSource source = type.getProtectionDomain().getCodeSource(); + if (source == null || source.getLocation() == null) { + return null; + } + try { + return Path.of(source.getLocation().toURI()); + } + catch (URISyntaxException | IllegalArgumentException notAFile) { + return null; + } + } +} diff --git a/core/grantforge-plugin-host/src/main/java/org/devlive/grantforge/plugin/host/PluginHostConfiguration.java b/core/grantforge-plugin-host/src/main/java/org/devlive/grantforge/plugin/host/PluginHostConfiguration.java index e1f717f6..fc043cd6 100644 --- a/core/grantforge-plugin-host/src/main/java/org/devlive/grantforge/plugin/host/PluginHostConfiguration.java +++ b/core/grantforge-plugin-host/src/main/java/org/devlive/grantforge/plugin/host/PluginHostConfiguration.java @@ -5,6 +5,7 @@ package org.devlive.grantforge.plugin.host; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.ApplicationRunner; import org.springframework.context.annotation.Bean; @@ -15,8 +16,9 @@ /** * The plugin host: plugins are looked up at start-up from the classpath and from - * {@code grantforge.plugins.directory} (default {@code plugins}, relative to the working directory); calls into - * plugins may take {@code grantforge.plugins.call-timeout} (default 10 seconds). + * {@code grantforge.plugins.directory} (default {@code plugins}, relative to the working directory, or the repository's + * plugin modules when the server runs from the sources; see {@link PluginDirectory}); calls into plugins may take + * {@code grantforge.plugins.call-timeout} (default 10 seconds). */ @Configuration(proxyBeanMethods = false) public class PluginHostConfiguration @@ -36,7 +38,7 @@ public PluginCalls pluginCalls(@Value("${grantforge.plugins.call-timeout:10s}") /** * The installed plugins. * - * @param directory the plugins directory + * @param directory the plugins directory as configured; blank to pick it (see {@link PluginDirectory}) * @param switches which plugins are switched off * @param calls calls plugin code with a time limit * @return the registry, empty until the start-up scan @@ -44,10 +46,12 @@ public PluginCalls pluginCalls(@Value("${grantforge.plugins.call-timeout:10s}") @Bean(destroyMethod = "close") // The plugin API that plugins share must come from the loader that loaded the server, not a thread's. @SuppressWarnings("PMD.UseProperClassLoader") - public PluginRegistry pluginRegistry(@Value("${grantforge.plugins.directory:plugins}") Path directory, PluginSwitches switches, + public PluginRegistry pluginRegistry(@Value("${grantforge.plugins.directory:}") String directory, PluginSwitches switches, PluginCalls calls) { - return new PluginRegistry(directory, switches, calls, PluginHostConfiguration.class.getClassLoader()); + Path chosen = PluginDirectory.choose(directory, PluginDirectory.codeSource(PluginHostConfiguration.class)); + LoggerFactory.getLogger(PluginHostConfiguration.class).info("Plugins directory: {}", chosen.toAbsolutePath()); + return new PluginRegistry(chosen, switches, calls, PluginHostConfiguration.class.getClassLoader()); } /** diff --git a/core/grantforge-plugin-host/src/main/java/org/devlive/grantforge/plugin/host/PluginPackage.java b/core/grantforge-plugin-host/src/main/java/org/devlive/grantforge/plugin/host/PluginPackage.java index 23c39184..35e4da25 100644 --- a/core/grantforge-plugin-host/src/main/java/org/devlive/grantforge/plugin/host/PluginPackage.java +++ b/core/grantforge-plugin-host/src/main/java/org/devlive/grantforge/plugin/host/PluginPackage.java @@ -38,12 +38,22 @@ * their dependencies) in {@code lib/} or at the top. Zips are unpacked into the directory's {@code .work} * folder. * + *

A plugin's Maven module counts too once it is built, so a server started from the sources loads the plugins of + * the repository (D-89): its classes come from {@code target/classes} and its dependencies from + * {@code target/plugin-lib}, which the plugin's build copies there. + * * @param location the file or directory name in the plugins directory * @param descriptor what the plugin says about itself * @param urls what its class loader loads from */ public record PluginPackage(String location, PluginDescriptor descriptor, List urls) { + /** Where a plugin module's build puts its classes, descriptor included. */ + static final String MODULE_CLASSES = "target/classes"; + + /** Where a plugin module's build copies its dependencies. */ + static final String MODULE_LIB = "target/plugin-lib"; + /** The folder of the plugins directory zips are unpacked into; never scanned for plugins itself. */ public static final String WORK = ".work"; @@ -60,8 +70,7 @@ public record PluginPackage(String location, PluginDescriptor descriptor, List urls = new ArrayList<>(); + urls.add(classes.toUri().toURL()); + Path lib = module.resolve(MODULE_LIB); + if (Files.isDirectory(lib)) { + urls.addAll(jars(lib)); + } + return new PluginPackage(location, descriptor, urls); + } + private static PluginPackage directory(String location, Path root) throws IOException { diff --git a/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/PluginDirectoryTest.java b/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/PluginDirectoryTest.java new file mode 100644 index 00000000..fddd2c69 --- /dev/null +++ b/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/PluginDirectoryTest.java @@ -0,0 +1,58 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.plugin.host; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +class PluginDirectoryTest +{ + @TempDir + private Path root; + + @Test + void anExplicitDirectoryWins() + { + assertThat(PluginDirectory.choose(" /opt/plugins ", root)).isEqualTo(Path.of("/opt/plugins")); + } + + @Test + void aServerRunFromTheSourcesUsesTheRepositorysPluginModules() throws IOException + { + Files.writeString(root.resolve("pom.xml"), ""); + Files.createDirectories(root.resolve("core")); + Path plugins = Files.createDirectories(root.resolve("plugins")); + Path classes = Files.createDirectories(root.resolve("core/grantforge-plugin-host/target/classes")); + + assertThat(PluginDirectory.sourceTree(classes)).isEqualTo(plugins); + // The working directory of the tests has no plugins folder, as an IDE's run of the server may not. + assertThat(PluginDirectory.choose("", classes)).isEqualTo(plugins); + } + + @Test + void aReleaseRunsFromJarsAndKeepsThePluginsFolder() throws IOException + { + Path jar = Files.writeString(root.resolve("grantforge-plugin-host.jar"), ""); + + assertThat(PluginDirectory.sourceTree(jar)).isNull(); + assertThat(PluginDirectory.sourceTree(null)).isNull(); + assertThat(PluginDirectory.sourceTree(Files.createDirectories(root.resolve("elsewhere")))).isNull(); + assertThat(PluginDirectory.choose("", jar)).isEqualTo(Path.of(PluginDirectory.STANDARD)); + } + + @Test + void findsWhereClassesCameFrom() + { + assertThat(PluginDirectory.codeSource(PluginDirectory.class)).isDirectory(); + assertThat(PluginDirectory.codeSource(String.class)).isNull(); + } +} diff --git a/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/PluginHostConfigurationTest.java b/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/PluginHostConfigurationTest.java index 63278b21..0ce18dd0 100644 --- a/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/PluginHostConfigurationTest.java +++ b/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/PluginHostConfigurationTest.java @@ -26,7 +26,7 @@ void createsTheCallerAndTheRegistryAndScansAtStartUp() { PluginHostConfiguration configuration = new PluginHostConfiguration(); try (PluginCalls calls = configuration.pluginCalls(Duration.ofSeconds(1)); - PluginRegistry registry = configuration.pluginRegistry(plugins, new PluginSwitches() + PluginRegistry registry = configuration.pluginRegistry(plugins.toString(), new PluginSwitches() { @Override public boolean enabled(String pluginId) diff --git a/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/PluginPackageTest.java b/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/PluginPackageTest.java index 20ecafaa..5b645d79 100644 --- a/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/PluginPackageTest.java +++ b/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/PluginPackageTest.java @@ -37,6 +37,38 @@ void candidatesAreJarsZipsAndDirectoriesButNotHiddenOnes() assertThat(PluginPackage.candidate(source)).isFalse(); Files.writeString(source.resolve(PluginDescriptor.FILE_NAME), "id: example"); assertThat(PluginPackage.candidate(source)).isTrue(); + // A module counts once its build put the descriptor into target/classes. + Path module = Files.createDirectories(root.resolve("grantforge-plugin-hdfs")); + Files.writeString(module.resolve("pom.xml"), ""); + assertThat(PluginPackage.candidate(module)).isFalse(); + Files.writeString(Files.createDirectories(module.resolve(PluginPackage.MODULE_CLASSES)).resolve(PluginDescriptor.FILE_NAME), "id: hdfs"); + assertThat(PluginPackage.candidate(module)).isTrue(); + } + + @Test + void readsABuiltModuleWithTheDependenciesItsBuildCopied() + throws IOException + { + Path module = Files.createDirectories(root.resolve("grantforge-plugin-hdfs")); + Files.writeString(module.resolve("pom.xml"), ""); + Path classes = Files.createDirectories(module.resolve(PluginPackage.MODULE_CLASSES)); + Files.writeString(classes.resolve(PluginDescriptor.FILE_NAME), """ + id: hdfs + version: 1.0.0 + name: HDFS + apiVersion: 1.0 + providers: org.example.HdfsProvider + """); + + assertThat(PluginPackage.read(module, root.resolve(PluginPackage.WORK)).urls()).extracting(url -> url.getPath()) + .singleElement().asString().endsWith("target/classes/"); + Path lib = Files.createDirectories(module.resolve(PluginPackage.MODULE_LIB)); + Files.writeString(lib.resolve("hadoop-client-api.jar"), ""); + Files.writeString(lib.resolve("notes.txt"), ""); + PluginPackage read = PluginPackage.read(module, root.resolve(PluginPackage.WORK)); + assertThat(read.location()).isEqualTo("grantforge-plugin-hdfs"); + assertThat(read.descriptor().id()).isEqualTo("hdfs"); + assertThat(read.urls()).extracting(url -> url.getPath().replaceAll(".*/(?=.)", "")).containsExactly("classes/", "hadoop-client-api.jar"); } @Test diff --git a/core/grantforge-server/src/test/resources/config/application.properties b/core/grantforge-server/src/test/resources/config/application.properties new file mode 100644 index 00000000..f2700897 --- /dev/null +++ b/core/grantforge-server/src/test/resources/config/application.properties @@ -0,0 +1,8 @@ +# Copyright (c) 2026 devlive-community/grantforge +# +# Licensed under the MIT License. See the LICENSE file in the +# project root for full license text. + +# Tests run from build output folders, which would make the plugin host load the repository's built plugin modules +# (D-89); tests that want plugins install their own. +grantforge.plugins.directory=target/no-plugins-here diff --git a/core/grantforge-service/src/test/resources/config/application.properties b/core/grantforge-service/src/test/resources/config/application.properties new file mode 100644 index 00000000..f2700897 --- /dev/null +++ b/core/grantforge-service/src/test/resources/config/application.properties @@ -0,0 +1,8 @@ +# Copyright (c) 2026 devlive-community/grantforge +# +# Licensed under the MIT License. See the LICENSE file in the +# project root for full license text. + +# Tests run from build output folders, which would make the plugin host load the repository's built plugin modules +# (D-89); tests that want plugins install their own. +grantforge.plugins.directory=target/no-plugins-here diff --git a/docs/content/architecture/development.md b/docs/content/architecture/development.md index fa059cc7..58b88b60 100644 --- a/docs/content/architecture/development.md +++ b/docs/content/architecture/development.md @@ -30,6 +30,10 @@ bash script/ci/perf_benchmark.sh smoke # 小规模性能基准 控制台开发时运行 `pnpm dev`（`core/grantforge-web`），Vite 把 `/api` 等请求代理到 `localhost:9999` 的服务。 +## 在 IDE 中启动 + +直接运行 `org.devlive.grantforge.server.GrantForge`（模块 `grantforge-server`），默认使用 H2 数据库。服务端会自动加载仓库 `plugins/` 下构建过的插件模块（见 [插件与服务类型](/architecture/plugins/)）；插件模块第一次使用前执行一次 `./mvnw -pl plugins/grantforge-plugin-hdfs -am install -DskipTests` 复制它的依赖。 + ## 代码规范 - 后端：Error Prone + NullAway（默认非空，可空处用 JSpecify `@Nullable`）、Checkstyle、PMD、SpotBugs；ArchUnit 守护共同的约定（不用字段注入、不写原生 SQL、实体不出现在 API 中、不允许 `Optional.get()` 等）。 diff --git a/docs/content/architecture/plugins.md b/docs/content/architecture/plugins.md index 3c90e1c3..2873fe9f 100644 --- a/docs/content/architecture/plugins.md +++ b/docs/content/architecture/plugins.md @@ -100,6 +100,33 @@ providers: 把它放进 `grantforge.plugins.directory`（默认 `plugins`），在控制台的“插件”页点击重新扫描即可，无需重启。 +插件带有依赖时，像 `plugins/grantforge-plugin-hdfs` 那样用 assembly 打成 `plugin` 分类的 zip（描述符在顶层、`classes/`、`lib/`），并在 `generate-resources` 阶段把运行时依赖复制到 `target/plugin-lib`： + +```xml + + maven-dependency-plugin + + + plugin-lib + generate-resources + copy-dependencies + + runtime + ${project.build.directory}/plugin-lib + + + + +``` + +## 从源码启动时 + +在 IDE 里直接启动 `org.devlive.grantforge.server.GrantForge` 时，服务端的类来自各模块的 `target/classes`，此时如果没有配置 `grantforge.plugins.directory`、工作目录下也没有 `plugins` 目录，就使用仓库的 `plugins/` 目录：其中构建过的插件模块（`target/classes` 里有描述符）直接作为插件加载，类来自 `target/classes`，依赖来自 `target/plugin-lib`。修改插件代码后由 IDE 重新编译，在控制台“插件”页重新扫描即可生效。插件模块第一次使用前，用 Maven 构建一次以复制依赖： + +```bash +./mvnw -pl plugins/grantforge-plugin-hdfs -am install -DskipTests +``` + ## 兼容性 `apiVersion` 声明插件需要的契约版本。宿主当前提供 `1.0.0`，主版本相同且不低于所需版本的插件才会加载，否则标为“不兼容”。契约的每次变化都会提升版本，CI 用 japicmp 与上一个发行版比较（`script/ci/check_plugin_api_compat.py`），不兼容的改动必须提升主版本。 From 02bd6d1ad66e9550252bcbb92f4c813e1c7614a2 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 02:09:54 -0400 Subject: [PATCH 08/22] feat(hdfs): talk to clusters with Hadoop's client like Apache Ranger The HDFS service type used WebHDFS only. It now uses Hadoop's shaded client, so a service names its cluster as Hadoop does (hdfs:// with HA nameservices, webhdfs://, swebhdfs://, viewfs://), takes Ranger's settings under their names, and signs the lookup user in with simple authentication or Kerberos by keytab or password. A real KDC tests the Kerberos logins. --- .../plugin/host/HdfsPluginTest.java | 86 ++---- docs/content/guide/data-services.md | 20 +- plugins/grantforge-plugin-hdfs/pom.xml | 50 +++- .../devlive/grantforge/hdfs/HadoopClient.java | 246 +++++++++++++++++ .../devlive/grantforge/hdfs/HdfsProvider.java | 152 ++++++++--- .../org/devlive/grantforge/hdfs/WebHdfs.java | 254 ------------------ .../devlive/grantforge/hdfs/FakeWebHdfs.java | 20 +- .../grantforge/hdfs/HadoopClientTest.java | 52 ++++ .../grantforge/hdfs/HdfsProviderTest.java | 97 ++++--- .../grantforge/hdfs/KerberosLoginTest.java | 110 ++++++++ .../devlive/grantforge/hdfs/WebHdfsTest.java | 112 -------- pom.xml | 25 ++ 12 files changed, 702 insertions(+), 522 deletions(-) create mode 100644 plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HadoopClient.java delete mode 100644 plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/WebHdfs.java create mode 100644 plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HadoopClientTest.java create mode 100644 plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/KerberosLoginTest.java delete mode 100644 plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/WebHdfsTest.java diff --git a/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/HdfsPluginTest.java b/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/HdfsPluginTest.java index 670abbc4..3a85e66c 100644 --- a/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/HdfsPluginTest.java +++ b/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/HdfsPluginTest.java @@ -5,90 +5,37 @@ package org.devlive.grantforge.plugin.host; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.sun.net.httpserver.HttpServer; -import org.devlive.grantforge.hdfs.HdfsProvider; import org.devlive.grantforge.plugin.api.ConnectionResult; import org.devlive.grantforge.plugin.api.LookupRequest; -import org.devlive.grantforge.plugin.api.PluginDescriptor; import org.devlive.grantforge.plugin.api.ServiceConfig; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import tools.jackson.core.JsonParser; -import tools.jackson.databind.json.JsonMapper; -import java.io.IOException; -import java.io.OutputStream; -import java.net.InetSocketAddress; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.time.Duration; import java.util.List; import java.util.Map; -import java.util.stream.Stream; +import static java.util.Objects.requireNonNull; import static org.assertj.core.api.Assertions.assertThat; /** - * Installs the HDFS plugin as the release ships it, a directory with its classes and its own Jackson in lib/, and talks - * to a WebHDFS endpoint through it: the plugin must work in its own class loader, which sees none of the server's - * libraries. + * Loads the HDFS plugin the way a server started from the sources does (D-89): straight from its built module in the + * repository's plugins folder, with Hadoop's client from the module's plugin-lib, in a class loader that sees none of + * the server's libraries; then lists a folder through Hadoop's file system. The module is built first, as this one + * depends on it. */ class HdfsPluginTest { @TempDir - private Path plugins; - - private static Path location(Class type) throws URISyntaxException - { - return Path.of(type.getProtectionDomain().getCodeSource().getLocation().toURI()); - } - - private void install() throws IOException, URISyntaxException - { - Path target = plugins.resolve("hdfs"); - Path built = location(HdfsProvider.class); - Files.createDirectories(target.resolve("lib")); - for (Class library : List.of(JsonMapper.class, JsonParser.class, JsonProperty.class)) { - Path jar = location(library); - Files.copy(jar, target.resolve("lib").resolve(jar.getFileName().toString()), StandardCopyOption.REPLACE_EXISTING); - } - if (Files.isRegularFile(built)) { - Files.copy(built, target.resolve("lib").resolve("grantforge-plugin-hdfs.jar")); - try (var jar = new java.util.jar.JarFile(built.toFile())) { - Files.copy(jar.getInputStream(jar.getEntry(PluginDescriptor.FILE_NAME)), target.resolve(PluginDescriptor.FILE_NAME)); - } - return; - } - Files.copy(built.resolve(PluginDescriptor.FILE_NAME), target.resolve(PluginDescriptor.FILE_NAME)); - try (Stream files = Files.walk(built)) { - for (Path file : files.filter(Files::isRegularFile).toList()) { - Path copy = target.resolve("classes").resolve(built.relativize(file).toString()); - Files.createDirectories(copy.getParent()); - Files.copy(file, copy, StandardCopyOption.REPLACE_EXISTING); - } - } - } + private Path files; @Test - void loadsWithItsOwnJacksonAndReachesWebHdfs() throws Exception + void loadsTheBuiltModuleWithHadoopsClient() throws Exception { - install(); - HttpServer namenode = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); - namenode.createContext("/webhdfs/v1", exchange -> { - String body = exchange.getRequestURI().getQuery().contains("LISTSTATUS") - ? "{\"FileStatuses\":{\"FileStatus\":[{\"pathSuffix\":\"user\",\"type\":\"DIRECTORY\"}]}}" - : "{\"FileStatus\":{\"pathSuffix\":\"\",\"type\":\"DIRECTORY\"}}"; - byte[] bytes = body.getBytes(StandardCharsets.UTF_8); - exchange.sendResponseHeaders(200, bytes.length); - try (OutputStream out = exchange.getResponseBody()) { - out.write(bytes); - } - }); - namenode.start(); + Path plugins = requireNonNull(PluginDirectory.sourceTree(PluginDirectory.codeSource(HdfsPluginTest.class)), "no repository"); + Files.createDirectories(files.resolve("data/sales")); PluginSwitches on = new PluginSwitches() { @Override @@ -103,23 +50,22 @@ public void set(String pluginId, boolean enabled) // Always on. } }; - ServiceConfig config = new ServiceConfig("lake", Map.of("url", "http://127.0.0.1:" + namenode.getAddress().getPort())); - try (PluginCalls calls = new PluginCalls(Duration.ofSeconds(10)); + // A local file system stands in for a cluster: the same Hadoop client code lists it. + ServiceConfig config = new ServiceConfig("lake", Map.of("fs.default.name", "file:///", "username", "hdfs")); + try (PluginCalls calls = new PluginCalls(Duration.ofSeconds(30)); PluginRegistry registry = new PluginRegistry(plugins, on, calls, HdfsPluginTest.class.getClassLoader())) { registry.scan(); InstalledPlugin hdfs = registry.plugins().stream().filter(plugin -> plugin.id().equals("hdfs")).findFirst().orElseThrow(); assertThat(hdfs.status()).as(String.valueOf(hdfs.problem())).isEqualTo(PluginStatus.ACTIVE); - assertThat(registry.serviceType("hdfs")).map(type -> type.label()).contains("HDFS"); + assertThat(hdfs.location()).isEqualTo("grantforge-plugin-hdfs"); ConnectionResult connected = registry.call("hdfs", provider -> { assertThat(provider.getClass().getClassLoader()).isInstanceOf(PluginClassLoader.class); return provider.testConnection(config); }); assertThat(connected.status()).as(String.valueOf(connected.message())).isEqualTo(ConnectionResult.Status.SUCCEEDED); - List found = registry.call("hdfs", provider -> provider.lookup(new LookupRequest(config, "path", "/u", Map.of(), 10))); - assertThat(found).containsExactly("/user"); - } - finally { - namenode.stop(0); + String data = files.resolve("data").toString(); + List found = registry.call("hdfs", provider -> provider.lookup(new LookupRequest(config, "path", data + "/s", Map.of(), 10))); + assertThat(found).containsExactly(data + "/sales"); } } } diff --git a/docs/content/guide/data-services.md b/docs/content/guide/data-services.md index 949dea83..13fa1e7e 100644 --- a/docs/content/guide/data-services.md +++ b/docs/content/guide/data-services.md @@ -30,14 +30,24 @@ flowchart LR ## HDFS -发行包自带 HDFS 插件（`plugins/hdfs`），服务类型 `hdfs`： +发行包自带 HDFS 插件（`plugins/hdfs`），服务类型 `hdfs`，与 Apache Ranger 的 HDFS 服务一致： - 资源只有一级 `path`，按路径匹配：`/data/sales` 匹配它本身，勾选“递归”后也匹配其下的全部文件与目录；支持排除。 - 访问类型 `read`、`write`、`execute`，与 HDFS 的权限位对应。 -- 配置：WebHDFS 地址（NameNode 的 HTTP 地址，如 `http://namenode:9870`；高可用时用逗号分隔两个 NameNode，自动找到 active 的那个；也可以填 HttpFS 地址）、查询目录用的用户（默认 `hdfs`，简单认证）与超时时间。 -- “测试连接”读取根目录的状态；写策略时输入路径会列出对应目录下的子目录与文件供选择。 - -插件通过 WebHDFS 的 REST 接口访问 HDFS，不依赖 Hadoop 客户端，适用于各个 Hadoop 版本。开启 Kerberos 的集群暂不支持测试连接与路径补全，策略仍可手工填写。 +- 插件用 Hadoop 自己的客户端连接集群，测试连接读取根目录，写策略时输入路径会列出对应目录下的子目录与文件。 + +| 配置 | 说明 | +| --- | --- | +| `username` | 查询目录用的用户；Kerberos 时为 principal，如 `grantforge@EXAMPLE.COM` | +| `password` / `keytab` | Kerberos 时二选一：principal 的密码，或 GrantForge 服务器上 keytab 文件的路径 | +| `fs.default.name` | `hdfs://namenode:8020`、高可用的 `hdfs://nameservice1`，或 `webhdfs://namenode:9870` | +| `hadoop.security.authentication` | `simple` 或 `kerberos` | +| `hadoop.security.authorization`、`hadoop.security.auth_to_local` | 与集群的 core-site.xml 一致 | +| `dfs.namenode.kerberos.principal` 等 | NameNode、DataNode、Secondary NameNode 的 principal，如 `nn/_HOST@EXAMPLE.COM` | +| `hadoop.rpc.protection` | `authentication`、`integrity` 或 `privacy`，与集群一致 | +| 附加 Hadoop 配置 | 每行一个 `key=value`，用于高可用等其他配置，例如 `dfs.nameservices=nameservice1`、`dfs.ha.namenodes.nameservice1=nn1,nn2`、`dfs.namenode.rpc-address.nameservice1.nn1=nn1:8020`、`dfs.client.failover.proxy.provider.nameservice1=org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider` | + +使用 Kerberos 时，GrantForge 服务器需要能找到 KDC：配置 `/etc/krb5.conf`，或用 `-Djava.security.krb5.conf=` 指定。 ## 数据服务 diff --git a/plugins/grantforge-plugin-hdfs/pom.xml b/plugins/grantforge-plugin-hdfs/pom.xml index 3462d8bc..bf131a46 100644 --- a/plugins/grantforge-plugin-hdfs/pom.xml +++ b/plugins/grantforge-plugin-hdfs/pom.xml @@ -20,9 +20,10 @@ grantforge-plugin-hdfs GrantForge HDFS Plugin - The HDFS service type (M13-03): paths with read, write and execute, looked up and tested through WebHDFS or - HttpFS over HTTP, so it works with every Hadoop version without the Hadoop client. Packaged as a plugin - directory (descriptor, classes and lib/) that the release ships in plugins/hdfs. + The HDFS service type (M13-03), modelled on Apache Ranger's: paths with read, write and execute, looked up and + tested with Hadoop's own (shaded) client over hdfs:// RPC, HA nameservices or webhdfs://, with simple or + Kerberos authentication. Packaged as a plugin directory (descriptor, classes and lib/) that the release ships + in plugins/hdfs. @@ -31,10 +32,16 @@ grantforge-plugin-api provided - + - tools.jackson.core - jackson-databind + org.apache.hadoop + hadoop-client-api + + + org.apache.hadoop + hadoop-client-runtime org.jspecify @@ -47,10 +54,41 @@ spring-boot-starter-test test + + org.apache.kerby + kerb-simplekdc + test + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + plugin-lib + generate-resources + + copy-dependencies + + + runtime + ${project.build.directory}/plugin-lib + + + + org.apache.maven.plugins maven-assembly-plugin diff --git a/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HadoopClient.java b/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HadoopClient.java new file mode 100644 index 00000000..c7eff9a0 --- /dev/null +++ b/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HadoopClient.java @@ -0,0 +1,246 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.security.UserGroupInformation; +import org.devlive.grantforge.plugin.api.ServiceConfig; +import org.jspecify.annotations.Nullable; + +import javax.security.auth.Subject; +import javax.security.auth.callback.Callback; +import javax.security.auth.callback.NameCallback; +import javax.security.auth.callback.PasswordCallback; +import javax.security.auth.callback.UnsupportedCallbackException; +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.LoginContext; +import javax.security.auth.login.LoginException; + +import java.io.IOException; +import java.lang.reflect.UndeclaredThrowableException; +import java.net.URI; +import java.security.PrivilegedExceptionAction; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.locks.ReentrantLock; + +import static java.util.Objects.requireNonNull; + +/** + * Talks to a cluster with Hadoop's own client, as Apache Ranger's HDFS service does: the service's settings become a + * Hadoop configuration, the lookup user signs in with simple authentication or Kerberos (keytab or password), and the + * file system is used as that user, then closed. Each call builds its own file system, so services never share one. + */ +final class HadoopClient +{ + /** Hadoop keeps the Kerberos settings in static state; logins are done one at a time. */ + private static final ReentrantLock LOGIN = new ReentrantLock(); + + /** Settings that make an unreachable cluster fail within the plugin call's time limit instead of retrying. */ + private static final Map FAIL_FAST = Map.of( + "ipc.client.connect.timeout", "5000", + "ipc.client.connect.max.retries", "1", + "ipc.client.connect.max.retries.on.timeouts", "1", + "ipc.client.rpc-timeout.ms", "8000", + "dfs.client.failover.max.attempts", "2", + "dfs.client.retry.policy.enabled", "false", + "dfs.webhdfs.socket.connect-timeout", "5s", + "dfs.webhdfs.socket.read-timeout", "8s", + "fs.hdfs.impl.disable.cache", "true", + "fs.webhdfs.impl.disable.cache", "true"); + + private final ServiceConfig config; + + HadoopClient(ServiceConfig config) + { + this.config = config; + } + + /** + * The Hadoop configuration of the service: fast-failing defaults, the service's named settings, then its additional + * properties, which may override anything. + */ + Configuration configuration() + { + Configuration hadoop = new Configuration(); + FAIL_FAST.forEach(hadoop::set); + hadoop.set("fs.defaultFS", config.require(HdfsProvider.DEFAULT_FS)); + for (String name : HdfsProvider.HADOOP_SETTINGS) { + String value = config.get(name); + if (value != null && !value.isBlank()) { + hadoop.set(name, value.strip()); + } + } + properties(config.get(HdfsProvider.EXTRA)).forEach(hadoop::set); + return hadoop; + } + + /** + * Reads additional properties, one {@code key=value} per line; blank lines and lines starting with {@code #} are + * skipped. + * + * @param text the lines, or {@code null} + * @return the properties in their order + * @throws IllegalArgumentException for a line that is not {@code key=value} + */ + static Map properties(@Nullable String text) + { + Map properties = new LinkedHashMap<>(); + if (text == null) { + return properties; + } + int number = 0; + for (String line : text.split("\\R")) { + number++; + String trimmed = line.strip(); + if (trimmed.isEmpty() || trimmed.startsWith("#")) { + continue; + } + int equals = trimmed.indexOf('='); + if (equals <= 0) { + throw new IllegalArgumentException("line " + number + " is not key=value"); + } + properties.put(trimmed.substring(0, equals).strip(), trimmed.substring(equals + 1).strip()); + } + return properties; + } + + /** + * Runs an action on the file system as the lookup user. + * + * @param action the action + * @param what it returns + * @return what it returned + * @throws IOException if signing in, reaching the cluster or the action fails + */ + // Restores the caller's interrupt when the call is interrupted; rethrows the action's own failure, not the wrapper + // doAs puts around it. + @SuppressWarnings({"PMD.DoNotUseThreads", "PMD.PreserveStackTrace"}) + T run(FileSystemAction action) throws IOException + { + Configuration hadoop = configuration(); + UserGroupInformation user = login(hadoop); + URI address = FileSystem.getDefaultUri(hadoop); + try { + return user.doAs((PrivilegedExceptionAction) () -> { + try (FileSystem files = FileSystem.newInstance(address, hadoop)) { + return action.apply(files); + } + }); + } + catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IOException("interrupted while reaching " + address, interrupted); + } + catch (UndeclaredThrowableException wrapped) { + if (wrapped.getCause() instanceof IOException failure) { + throw failure; + } + throw new IOException(String.valueOf(wrapped.getCause()), wrapped); + } + } + + private UserGroupInformation login(Configuration hadoop) throws IOException + { + String user = config.require(HdfsProvider.USER).strip(); + if (!HdfsProvider.KERBEROS.equals(config.get(HdfsProvider.AUTHENTICATION))) { + LOGIN.lock(); + try { + UserGroupInformation.setConfiguration(hadoop); + return UserGroupInformation.createRemoteUser(user); + } + finally { + LOGIN.unlock(); + } + } + String keytab = config.get(HdfsProvider.KEYTAB); + String password = config.get(HdfsProvider.PASSWORD); + String keytabPath = keytab == null || keytab.isBlank() ? null : keytab.strip(); + boolean withKeytab = keytabPath != null; + if (!withKeytab && (password == null || password.isEmpty())) { + throw new IOException("Kerberos needs the lookup user's password or a keytab"); + } + LOGIN.lock(); + try { + try { + UserGroupInformation.setConfiguration(hadoop); + } + catch (IllegalArgumentException unconfigured) { + throw new IOException("Kerberos is not set up on the GrantForge server (krb5.conf, or java.security.krb5.realm and" + + " .kdc): " + unconfigured.getMessage(), unconfigured); + } + UserGroupInformation signedIn = keytabPath != null ? UserGroupInformation.loginUserFromKeytabAndReturnUGI(user, keytabPath) + : UserGroupInformation.getUGIFromSubject(passwordLogin(user, requireNonNull(password, "password"))); + // A local file system asks for no credentials; a cluster would refuse later, so tell now. + if (!signedIn.hasKerberosCredentials() || !user.equals(signedIn.getUserName())) { + throw new IOException("Kerberos gave no credentials of " + user + (keytabPath == null ? "" : "; is it in " + keytabPath + "?")); + } + return signedIn; + } + finally { + LOGIN.unlock(); + } + } + + /** Signs a principal in with its password through the JDK's Kerberos login module. */ + private static Subject passwordLogin(String principal, String password) throws IOException + { + Map options = Map.of("useTicketCache", "false", "refreshKrb5Config", "true", "storeKey", "true", + "doNotPrompt", "false", "principal", principal); + javax.security.auth.login.Configuration jaas = new javax.security.auth.login.Configuration() + { + @Override + public AppConfigurationEntry[] getAppConfigurationEntry(String name) + { + return new AppConfigurationEntry[] {new AppConfigurationEntry("com.sun.security.auth.module.Krb5LoginModule", + AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options)}; + } + }; + Subject subject = new Subject(); + try { + LoginContext context = new LoginContext("grantforge-hdfs", subject, callbacks -> answer(callbacks, principal, password), jaas); + context.login(); + return subject; + } + catch (LoginException refused) { + throw new IOException("Kerberos refused " + principal + ": " + refused.getMessage(), refused); + } + } + + private static void answer(Callback[] callbacks, String principal, String password) throws UnsupportedCallbackException + { + for (Callback callback : callbacks) { + if (callback instanceof NameCallback name) { + name.setName(principal); + } + else if (callback instanceof PasswordCallback secret) { + secret.setPassword(password.toCharArray()); + } + else { + throw new UnsupportedCallbackException(callback); + } + } + } + + /** + * Something done with the file system. + * + * @param what it returns + */ + @FunctionalInterface + interface FileSystemAction + { + /** + * Does it. + * + * @param files the file system + * @return the result + * @throws IOException if the file system fails + */ + T apply(FileSystem files) throws IOException; + } +} diff --git a/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HdfsProvider.java b/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HdfsProvider.java index c2921748..9b81b6e3 100644 --- a/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HdfsProvider.java +++ b/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HdfsProvider.java @@ -5,6 +5,9 @@ package org.devlive.grantforge.hdfs; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.ipc.RemoteException; import org.devlive.grantforge.plugin.api.ConnectionResult; import org.devlive.grantforge.plugin.api.LookupRequest; import org.devlive.grantforge.plugin.api.ServiceConfig; @@ -16,16 +19,24 @@ import org.devlive.grantforge.plugin.api.model.MatcherType; import org.devlive.grantforge.plugin.api.model.ResourceDefinition; import org.devlive.grantforge.plugin.api.model.ServiceTypeDefinition; +import org.jspecify.annotations.Nullable; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; -import java.time.Duration; +import java.net.URI; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; import java.util.List; +import java.util.Locale; +import java.util.Set; /** - * The HDFS service type: paths, matched as paths and optionally with everything below them, with the access types the - * HDFS agent checks, read, write and execute. Connections and path lookups go through WebHDFS or HttpFS. + * The HDFS service type, modelled on Apache Ranger's: paths, matched as paths and optionally with everything below them, + * with read, write and execute. A service names its cluster like Hadoop does, {@code hdfs://namenode:8020}, an HA + * nameservice ({@code hdfs://nameservice1} with the nameservice's properties), or {@code webhdfs://}; the lookup user + * signs in with simple authentication or Kerberos. Connections and path lookups go through Hadoop's own client. */ public final class HdfsProvider implements ServiceTypeProvider @@ -34,10 +45,26 @@ public final class HdfsProvider public static final String TYPE = "hdfs"; static final String PATH = "path"; - static final String URL = "url"; static final String USER = "username"; - static final String TIMEOUT = "timeout"; - static final long MAX_TIMEOUT = 120; + static final String PASSWORD = "password"; + static final String KEYTAB = "keytab"; + static final String DEFAULT_FS = "fs.default.name"; + static final String AUTHORIZATION = "hadoop.security.authorization"; + static final String AUTHENTICATION = "hadoop.security.authentication"; + static final String AUTH_TO_LOCAL = "hadoop.security.auth_to_local"; + static final String DATANODE_PRINCIPAL = "dfs.datanode.kerberos.principal"; + static final String NAMENODE_PRINCIPAL = "dfs.namenode.kerberos.principal"; + static final String SECONDARY_PRINCIPAL = "dfs.secondary.namenode.kerberos.principal"; + static final String RPC_PROTECTION = "hadoop.rpc.protection"; + static final String EXTRA = "hadoop.config"; + static final String SIMPLE = "simple"; + static final String KERBEROS = "kerberos"; + + /** Settings passed to Hadoop under their own names. */ + static final List HADOOP_SETTINGS = List.of(AUTHORIZATION, AUTHENTICATION, AUTH_TO_LOCAL, DATANODE_PRINCIPAL, + NAMENODE_PRINCIPAL, SECONDARY_PRINCIPAL, RPC_PROTECTION); + + private static final Set SCHEMES = Set.of("hdfs", "webhdfs", "swebhdfs", "viewfs"); @Override public ServiceTypeDefinition definition() @@ -47,13 +74,30 @@ public ServiceTypeDefinition definition() .recursiveSupported(true).excludesSupported(true).lookupSupported(true).validLeaf(true).build()) .accessTypes(AccessTypeDefinition.of("read", "Read"), AccessTypeDefinition.of("write", "Write"), AccessTypeDefinition.of("execute", "Execute")) - .configFields(ConfigField.builder(URL).label("WebHDFS address").type(ConfigFieldType.STRING).mandatory() - .pattern("https?://\\S+(\\s*,\\s*https?://\\S+)*") - .description("The NameNodes' HTTP addresses, comma-separated for high availability, or HttpFS;" - + " such as http://namenode:9870").build(), - ConfigField.builder(USER).label("User").type(ConfigFieldType.STRING).defaultValue("hdfs") - .description("Who lists paths, with simple authentication").build(), - ConfigField.builder(TIMEOUT).label("Timeout (seconds)").type(ConfigFieldType.INTEGER).defaultValue("10").build()) + .configFields( + ConfigField.builder(USER).label("Username").type(ConfigFieldType.STRING).mandatory() + .description("Who lists paths; with Kerberos, the principal, such as grantforge@EXAMPLE.COM").build(), + ConfigField.builder(PASSWORD).label("Password").type(ConfigFieldType.SECRET) + .description("The principal's Kerberos password, unless a keytab is given").build(), + ConfigField.builder(KEYTAB).label("Keytab").type(ConfigFieldType.STRING) + .description("Path of the principal's keytab on the GrantForge server").build(), + ConfigField.builder(DEFAULT_FS).label("Namenode URL").type(ConfigFieldType.STRING).mandatory() + .pattern("[A-Za-z][A-Za-z0-9+.-]*://\\S+") + .description("Such as hdfs://namenode:8020, hdfs://nameservice1 or webhdfs://namenode:9870").build(), + ConfigField.builder(AUTHORIZATION).label("Authorization enabled").type(ConfigFieldType.BOOLEAN) + .defaultValue("false").build(), + ConfigField.builder(AUTHENTICATION).label("Authentication type").type(ConfigFieldType.ENUM) + .options(SIMPLE, KERBEROS).defaultValue(SIMPLE).build(), + ConfigField.builder(AUTH_TO_LOCAL).label("Auth to local rules").type(ConfigFieldType.TEXT).build(), + ConfigField.builder(DATANODE_PRINCIPAL).label("DataNode principal").type(ConfigFieldType.STRING).build(), + ConfigField.builder(NAMENODE_PRINCIPAL).label("NameNode principal").type(ConfigFieldType.STRING).build(), + ConfigField.builder(SECONDARY_PRINCIPAL).label("Secondary NameNode principal").type(ConfigFieldType.STRING).build(), + ConfigField.builder(RPC_PROTECTION).label("RPC protection").type(ConfigFieldType.ENUM) + .options("authentication", "integrity", "privacy").defaultValue("authentication").build(), + ConfigField.builder(EXTRA).label("Additional Hadoop properties").type(ConfigFieldType.TEXT) + .description("One key=value per line, such as the HA nameservice: dfs.nameservices," + + " dfs.ha.namenodes., dfs.namenode.rpc-address..," + + " dfs.client.failover.proxy.provider.").build()) .build(); } @@ -61,32 +105,50 @@ public ServiceTypeDefinition definition() public List validateConfig(ServiceConfig config) { List problems = new ArrayList<>(); - String url = config.get(URL); - if (url != null) { - try { - WebHdfs.addresses(url); - } - catch (IllegalArgumentException invalid) { - problems.add(new ConfigProblem(URL, ConfigProblem.Reason.INVALID, String.valueOf(invalid.getMessage()))); + String address = config.get(DEFAULT_FS); + if (address != null) { + String scheme = scheme(address); + if (scheme == null || !SCHEMES.contains(scheme)) { + problems.add(new ConfigProblem(DEFAULT_FS, ConfigProblem.Reason.INVALID, "use hdfs://, webhdfs://, swebhdfs:// or viewfs://")); } } - long timeout = config.getLong(TIMEOUT, 10); - if (timeout < 1 || timeout > MAX_TIMEOUT) { - problems.add(new ConfigProblem(TIMEOUT, ConfigProblem.Reason.INVALID, "1 to " + MAX_TIMEOUT + " seconds")); + if (KERBEROS.equals(config.get(AUTHENTICATION)) && blank(config.get(PASSWORD)) && blank(config.get(KEYTAB))) { + problems.add(new ConfigProblem(PASSWORD, ConfigProblem.Reason.REQUIRED, "Kerberos needs a password or a keytab")); + } + try { + HadoopClient.properties(config.get(EXTRA)); + } + catch (IllegalArgumentException invalid) { + problems.add(new ConfigProblem(EXTRA, ConfigProblem.Reason.INVALID, invalid.getMessage())); } return problems; } + private static @Nullable String scheme(String address) + { + try { + String scheme = URI.create(address.strip()).getScheme(); + return scheme == null ? null : scheme.toLowerCase(Locale.ROOT); + } + catch (IllegalArgumentException invalid) { + return null; + } + } + + private static boolean blank(@Nullable String value) + { + return value == null || value.isBlank(); + } + @Override public ConnectionResult testConnection(ServiceConfig config) { try { - WebHdfs.Entry root = client(config).status("/"); - return root != null && root.directory() ? ConnectionResult.succeeded() - : ConnectionResult.failed("the root of the file system is not a directory"); + FileStatus root = new HadoopClient(config).run(files -> files.getFileStatus(new Path("/"))); + return root.isDirectory() ? ConnectionResult.succeeded() : ConnectionResult.failed("the root of the file system is not a directory"); } catch (IOException | IllegalArgumentException failed) { - return ConnectionResult.failed(String.valueOf(failed.getMessage())); + return ConnectionResult.failed(message(failed)); } } @@ -106,23 +168,37 @@ public List lookup(LookupRequest request) String directory = slash == 0 ? "/" : path.substring(0, slash); String prefix = path.substring(slash + 1); String base = "/".equals(directory) ? "" : directory; - List entries; + List entries; try { - entries = client(request.config()).list(directory); + entries = new HadoopClient(request.config()).run(files -> { + try { + return Arrays.asList(files.listStatus(new Path(directory))); + } + catch (FileNotFoundException missing) { + return List.of(); + } + catch (RemoteException remote) { + // An old or unusual NameNode may not name the Java class, so the client cannot unwrap it. + if (remote.getClassName() != null && remote.getClassName().endsWith("FileNotFoundException")) { + return List.of(); + } + throw remote; + } + }); } catch (IOException failed) { - throw new UncheckedIOException(failed); + throw new UncheckedIOException(message(failed), failed); } - return entries.stream().filter(entry -> entry.name().startsWith(prefix)) - .sorted((left, right) -> left.directory() == right.directory() ? left.name().compareTo(right.name()) - : left.directory() ? -1 : 1) - .limit(request.limit()).map(entry -> base + "/" + entry.name()).toList(); + return entries.stream().filter(entry -> entry.getPath().getName().startsWith(prefix)) + .sorted(Comparator.comparing((FileStatus entry) -> !entry.isDirectory()).thenComparing(entry -> entry.getPath().getName())) + .limit(request.limit()).map(entry -> base + "/" + entry.getPath().getName()).toList(); } - private static WebHdfs client(ServiceConfig config) + /** The first line of a Hadoop failure, which is the useful part of its often long message. */ + private static String message(Exception failure) { - String user = config.get(USER); - return new WebHdfs(WebHdfs.addresses(config.require(URL)), user == null || user.isBlank() ? "hdfs" : user.strip(), - Duration.ofSeconds(Math.max(1, Math.min(MAX_TIMEOUT, config.getLong(TIMEOUT, 10))))); + String text = failure.getMessage() == null ? failure.getClass().getSimpleName() : failure.getMessage(); + int line = text.indexOf('\n'); + return line < 0 ? text : text.substring(0, line); } } diff --git a/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/WebHdfs.java b/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/WebHdfs.java deleted file mode 100644 index af2a4ac1..00000000 --- a/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/WebHdfs.java +++ /dev/null @@ -1,254 +0,0 @@ -// Copyright (c) 2026 devlive-community/grantforge -// -// Licensed under the MIT License. See the LICENSE file in the -// project root for full license text. - -package org.devlive.grantforge.hdfs; - -import org.jspecify.annotations.Nullable; -import tools.jackson.core.JacksonException; -import tools.jackson.databind.JsonNode; -import tools.jackson.databind.json.JsonMapper; - -import java.io.IOException; -import java.net.URI; -import java.net.URLEncoder; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; - -import static java.util.Objects.requireNonNull; - -/** - * The few WebHDFS (or HttpFS) calls the service type needs, over the JDK's HTTP client with simple authentication - * ({@code user.name}). With several NameNode addresses, as in a high-availability pair, the calls go to the first one - * that answers as active: a standby NameNode refuses with a {@code StandbyException} and the next is asked. - */ -final class WebHdfs -{ - private static final JsonMapper JSON = JsonMapper.builder().build(); - private static final String PREFIX = "/webhdfs/v1"; - - private final List addresses; - private final String user; - private final Duration timeout; - private final HttpClient http; - - /** - * Creates the client. - * - * @param addresses the NameNode or HttpFS addresses, such as {@code http://namenode:9870} - * @param user the user to act as - * @param timeout how long a call may take - */ - WebHdfs(List addresses, String user, Duration timeout) - { - if (addresses.isEmpty()) { - throw new IllegalArgumentException("no WebHDFS address"); - } - this.addresses = List.copyOf(addresses); - this.user = requireNonNull(user, "user"); - this.timeout = requireNonNull(timeout, "timeout"); - this.http = HttpClient.newBuilder().connectTimeout(timeout).followRedirects(HttpClient.Redirect.NEVER).build(); - } - - /** - * Parses a comma-separated list of addresses. - * - * @param text the addresses - * @return the addresses, without their trailing slashes - * @throws IllegalArgumentException if one is not an http or https address - */ - static List addresses(String text) - { - List addresses = new ArrayList<>(); - for (String part : text.split(",")) { - String address = part.strip(); - if (address.isEmpty()) { - continue; - } - while (address.endsWith("/")) { - address = address.substring(0, address.length() - 1); - } - URI uri; - try { - uri = URI.create(address); - } - catch (IllegalArgumentException broken) { - throw new IllegalArgumentException(part.strip() + " is not an address", broken); - } - if (!("http".equals(uri.getScheme()) || "https".equals(uri.getScheme())) || uri.getHost() == null) { - throw new IllegalArgumentException(part.strip() + " is not an http or https address"); - } - addresses.add(uri); - } - if (addresses.isEmpty()) { - throw new IllegalArgumentException("no address given"); - } - return addresses; - } - - /** - * Returns the status of a path. - * - * @param path an absolute path - * @return the status, or {@code null} if the path does not exist - * @throws IOException if no NameNode answers, or one refuses - */ - @Nullable Entry status(String path) throws IOException - { - JsonNode answer = call(path, "GETFILESTATUS"); - return answer == null ? null : entry(answer.path("FileStatus")); - } - - /** - * Lists a directory. - * - * @param path an absolute path - * @return its entries, or an empty list if it does not exist - * @throws IOException if no NameNode answers, or one refuses - */ - List list(String path) throws IOException - { - JsonNode answer = call(path, "LISTSTATUS"); - if (answer == null) { - return List.of(); - } - List entries = new ArrayList<>(); - for (JsonNode status : answer.path("FileStatuses").path("FileStatus")) { - entries.add(entry(status)); - } - return entries; - } - - private static Entry entry(JsonNode status) - { - return new Entry(status.path("pathSuffix").asString(""), "DIRECTORY".equals(status.path("type").asString(""))); - } - - /** Calls the first NameNode that answers as active; {@code null} when the path does not exist. */ - private @Nullable JsonNode call(String path, String operation) throws IOException - { - IOException last = null; - for (URI address : addresses) { - try { - return call(address, path, operation); - } - catch (StandbyException standby) { - last = standby; - } - catch (Refused refused) { - throw refused; - } - catch (IOException unreachable) { - last = unreachable; - } - } - throw requireNonNull(last, "no address was asked"); - } - - // Restores the caller's interrupt when the call is interrupted. - @SuppressWarnings("PMD.DoNotUseThreads") - private @Nullable JsonNode call(URI address, String path, String operation) throws IOException - { - URI uri = URI.create(address + PREFIX + encode(path) + "?op=" + operation + "&user.name=" - + URLEncoder.encode(user, StandardCharsets.UTF_8)); - HttpRequest request = HttpRequest.newBuilder(uri).timeout(timeout).header("Accept", "application/json").GET().build(); - HttpResponse response; - try { - response = http.send(request, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); - } - catch (InterruptedException interrupted) { - Thread.currentThread().interrupt(); - throw new IOException("interrupted while asking " + address, interrupted); - } - catch (IOException unreachable) { - throw new IOException(address + " cannot be reached: " + unreachable.getMessage(), unreachable); - } - JsonNode body = json(response.body()); - if (response.statusCode() == 200) { - if (body == null) { - throw new IOException(address + " did not answer in JSON; is it a WebHDFS address?"); - } - return body; - } - JsonNode remote = body == null ? null : body.path("RemoteException"); - String exception = remote == null ? "" : remote.path("exception").asString(""); - String message = remote == null ? "" : remote.path("message").asString(""); - if ("StandbyException".equals(exception)) { - throw new StandbyException(address + " is a standby NameNode"); - } - if (response.statusCode() == 404 && ("FileNotFoundException".equals(exception) || exception.isEmpty())) { - if (remote == null || remote.isMissingNode()) { - throw new Refused(address + " answered 404; is it a WebHDFS address?"); - } - return null; - } - throw new Refused(address + " refused " + operation.toLowerCase(Locale.ROOT) + " " + path + " (" + response.statusCode() - + (exception.isEmpty() ? "" : " " + exception) + (message.isEmpty() ? "" : ": " + message) + ")"); - } - - private static @Nullable JsonNode json(String body) - { - try { - JsonNode node = JSON.readTree(body); - return node != null && node.isObject() ? node : null; - } - catch (JacksonException notJson) { - return null; - } - } - - /** Encodes each segment of a path for the URL, keeping the slashes. */ - static String encode(String path) - { - StringBuilder encoded = new StringBuilder(); - for (String segment : path.split("/", -1)) { - if (encoded.length() > 0 || !segment.isEmpty()) { - encoded.append('/'); - } - encoded.append(URLEncoder.encode(segment, StandardCharsets.UTF_8).replace("+", "%20")); - } - String result = encoded.toString(); - return result.startsWith("/") ? result : "/" + result; - } - - /** - * A directory entry. - * - * @param name its name within the directory - * @param directory whether it is a directory - */ - record Entry(String name, boolean directory) - { - } - - /** A NameNode that is standby, so the next one is asked. */ - static final class StandbyException - extends IOException - { - private static final long serialVersionUID = 1L; - - StandbyException(String message) - { - super(message); - } - } - - /** A NameNode that answered and refused, so asking another does not help. */ - static final class Refused - extends IOException - { - private static final long serialVersionUID = 1L; - - Refused(String message) - { - super(message); - } - } -} diff --git a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/FakeWebHdfs.java b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/FakeWebHdfs.java index 621669e3..943bf2eb 100644 --- a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/FakeWebHdfs.java +++ b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/FakeWebHdfs.java @@ -62,7 +62,8 @@ private void handle(HttpExchange exchange) throws IOException return; } if (query.contains("user.name=nobody")) { - respond(exchange, 403, "{\"RemoteException\":{\"exception\":\"AccessControlException\",\"message\":\"Permission denied: user=nobody\"}}"); + respond(exchange, 403, "{\"RemoteException\":{\"exception\":\"AccessControlException\",\"javaClassName\":" + + "\"org.apache.hadoop.security.AccessControlException\",\"message\":\"Permission denied: user=nobody\"}}"); return; } String normal = path.length() > 1 && path.endsWith("/") ? path.substring(0, path.length() - 1) : path; @@ -72,22 +73,29 @@ private void handle(HttpExchange exchange) throws IOException boolean directory = TREE.containsKey(normal); boolean file = "/user/notes.txt".equals(normal); if (!directory && !file) { - respond(exchange, 404, "{\"RemoteException\":{\"exception\":\"FileNotFoundException\",\"message\":\"File does not exist: " - + normal + "\"}}"); + respond(exchange, 404, "{\"RemoteException\":{\"exception\":\"FileNotFoundException\",\"javaClassName\":" + + "\"java.io.FileNotFoundException\",\"message\":\"File does not exist: " + normal + "\"}}"); return; } if (query.contains("op=GETFILESTATUS")) { - respond(exchange, 200, "{\"FileStatus\":{\"pathSuffix\":\"\",\"type\":\"" + (directory ? "DIRECTORY" : "FILE") + "\"}}"); + respond(exchange, 200, "{\"FileStatus\":" + status("", directory ? "DIRECTORY" : "FILE") + "}"); return; } StringBuilder statuses = new StringBuilder(); for (String[] entry : TREE.getOrDefault(normal, List.of())) { - statuses.append(statuses.length() == 0 ? "" : ",").append("{\"pathSuffix\":\"").append(entry[0]).append("\",\"type\":\"") - .append(entry[1]).append("\"}"); + statuses.append(statuses.length() == 0 ? "" : ",").append(status(entry[0], entry[1])); } respond(exchange, 200, "{\"FileStatuses\":{\"FileStatus\":[" + statuses + "]}}"); } + /** A FileStatus with every field Hadoop's WebHDFS client reads. */ + private static String status(String name, String type) + { + return "{\"pathSuffix\":\"" + name + "\",\"type\":\"" + type + "\",\"length\":0,\"owner\":\"hdfs\",\"group\":\"supergroup\"," + + "\"permission\":\"755\",\"accessTime\":0,\"modificationTime\":0,\"blockSize\":134217728,\"replication\":" + + ("FILE".equals(type) ? 3 : 0) + ",\"fileId\":16386,\"childrenNum\":0,\"storagePolicy\":0}"; + } + private static void respond(HttpExchange exchange, int status, String body) throws IOException { byte[] bytes = body.getBytes(StandardCharsets.UTF_8); diff --git a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HadoopClientTest.java b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HadoopClientTest.java new file mode 100644 index 00000000..d3c764f2 --- /dev/null +++ b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HadoopClientTest.java @@ -0,0 +1,52 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; + +class HadoopClientTest +{ + @Test + void readsAdditionalPropertiesLineByLine() + { + assertThat(HadoopClient.properties(null)).isEmpty(); + assertThat(HadoopClient.properties(""" + # the nameservice + dfs.nameservices = ns1 + + dfs.ha.namenodes.ns1=nn1,nn2 + dfs.namenode.rpc-address.ns1.nn1=a:8020=x + """)).containsExactly(Map.entry("dfs.nameservices", "ns1"), Map.entry("dfs.ha.namenodes.ns1", "nn1,nn2"), + Map.entry("dfs.namenode.rpc-address.ns1.nn1", "a:8020=x")); + assertThatIllegalArgumentException().isThrownBy(() -> HadoopClient.properties("a=b\n=c")).withMessageContaining("line 2"); + assertThatIllegalArgumentException().isThrownBy(() -> HadoopClient.properties("plain")).withMessageContaining("line 1"); + } + + @Test + void turnsTheServiceIntoAHadoopConfiguration() + { + Configuration hadoop = new HadoopClient(HdfsProviderTest.config("hdfs://ns1", "hadoop.security.authentication", "kerberos", + "dfs.namenode.kerberos.principal", " nn/_HOST@EXAMPLE.COM ", "hadoop.rpc.protection", "privacy", + "hadoop.security.auth_to_local", " ", "hadoop.config", "dfs.nameservices=ns1\nipc.client.connect.max.retries=5")) + .configuration(); + + assertThat(hadoop.get("fs.defaultFS")).isEqualTo("hdfs://ns1"); + assertThat(hadoop.get("hadoop.security.authentication")).isEqualTo("kerberos"); + assertThat(hadoop.get("dfs.namenode.kerberos.principal")).isEqualTo("nn/_HOST@EXAMPLE.COM"); + assertThat(hadoop.get("hadoop.rpc.protection")).isEqualTo("privacy"); + assertThat(hadoop.get("dfs.nameservices")).isEqualTo("ns1"); + // The service's own properties win over the fast-failing defaults. + assertThat(hadoop.get("ipc.client.connect.max.retries")).isEqualTo("5"); + assertThat(hadoop.get("ipc.client.connect.timeout")).isEqualTo("5000"); + assertThat(hadoop.get("fs.hdfs.impl.disable.cache")).isEqualTo("true"); + } +} diff --git a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HdfsProviderTest.java b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HdfsProviderTest.java index 8b3b6a7f..7bc3f748 100644 --- a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HdfsProviderTest.java +++ b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HdfsProviderTest.java @@ -13,9 +13,12 @@ import org.devlive.grantforge.plugin.api.model.ResourceDefinition; import org.devlive.grantforge.plugin.api.model.ServiceTypeDefinition; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -25,16 +28,20 @@ class HdfsProviderTest { + @TempDir + Path root; + private final HdfsProvider provider = new HdfsProvider(); - private static ServiceConfig config(String url, String... more) + static ServiceConfig config(String address, String... more) { Map values = new HashMap<>(); - values.put(HdfsProvider.URL, url); + values.put(HdfsProvider.DEFAULT_FS, address); + values.put(HdfsProvider.USER, "hdfs"); for (int index = 0; index < more.length; index += 2) { values.put(more[index], more[index + 1]); } - return new ServiceConfig("warehouse-hdfs", values); + return new ServiceConfig("lake", values); } private List lookup(ServiceConfig config, String typed) @@ -43,7 +50,7 @@ private List lookup(ServiceConfig config, String typed) } @Test - void declaresPathsWithReadWriteAndExecute() + void declaresPathsWithReadWriteAndExecuteAndRangersSettings() { ServiceTypeDefinition definition = provider.definition(); @@ -52,52 +59,80 @@ void declaresPathsWithReadWriteAndExecute() assertThat(path.name()).isEqualTo("path"); assertThat(path.matcher()).isEqualTo(MatcherType.PATH); assertThat(path.recursiveSupported()).isTrue(); + assertThat(path.excludesSupported()).isTrue(); assertThat(path.lookupSupported()).isTrue(); assertThat(path.caseSensitive()).isTrue(); assertThat(definition.accessTypes()).extracting(access -> access.name()).containsExactly("read", "write", "execute"); - assertThat(definition.configFields()).extracting(field -> field.name()).containsExactly("url", "username", "timeout"); + assertThat(definition.configFields()).extracting(field -> field.name()).containsExactly("username", "password", "keytab", + "fs.default.name", "hadoop.security.authorization", "hadoop.security.authentication", "hadoop.security.auth_to_local", + "dfs.datanode.kerberos.principal", "dfs.namenode.kerberos.principal", "dfs.secondary.namenode.kerberos.principal", + "hadoop.rpc.protection", "hadoop.config"); } @Test - void checksAddressesAndTimeouts() + void checksTheAddressKerberosCredentialsAndExtraProperties() { - assertThat(provider.validateConfig(config("http://nn1:9870,http://nn2:9870", "timeout", "30"))).isEmpty(); - assertThat(provider.validateConfig(config("ftp://nn1"))).extracting(ConfigProblem::field).containsExactly("url"); - assertThat(provider.validateConfig(config("http://nn1:9870", "timeout", "0"))).extracting(ConfigProblem::field) - .containsExactly("timeout"); - assertThat(provider.validateConfig(config("http://nn1:9870", "timeout", "121"))).extracting(ConfigProblem::reason) - .containsExactly(ConfigProblem.Reason.INVALID); - assertThat(provider.validateConfig(new ServiceConfig("warehouse-hdfs", Map.of()))).isEmpty(); + assertThat(provider.validateConfig(config("hdfs://nameservice1", "hadoop.config", "dfs.nameservices=nameservice1\n# note\n"))).isEmpty(); + assertThat(provider.validateConfig(config("webhdfs://namenode:9870"))).isEmpty(); + assertThat(provider.validateConfig(config("file:///tmp"))).extracting(ConfigProblem::field).containsExactly("fs.default.name"); + assertThat(provider.validateConfig(config("not an address"))).extracting(ConfigProblem::field).containsExactly("fs.default.name"); + assertThat(provider.validateConfig(config("hdfs://nn:8020", "hadoop.security.authentication", "kerberos"))) + .extracting(ConfigProblem::field, ConfigProblem::reason).containsExactly(org.assertj.core.groups.Tuple.tuple("password", + ConfigProblem.Reason.REQUIRED)); + assertThat(provider.validateConfig(config("hdfs://nn:8020", "hadoop.security.authentication", "kerberos", "keytab", "/etc/gf.keytab"))) + .isEmpty(); + assertThat(provider.validateConfig(config("hdfs://nn:8020", "hadoop.config", "no equals sign"))).extracting(ConfigProblem::field) + .containsExactly("hadoop.config"); + assertThat(provider.validateConfig(new ServiceConfig("lake", Map.of()))).isEmpty(); } @Test - void testsTheConnectionOnTheRoot() throws IOException + void testsTheConnectionAndLooksUpThroughHadoopsClient() throws IOException { - try (FakeWebHdfs namenode = new FakeWebHdfs(false)) { - assertThat(provider.testConnection(config(namenode.uri().toString()))).isEqualTo(ConnectionResult.succeeded()); - ConnectionResult refused = provider.testConnection(config(namenode.uri().toString(), "username", "nobody")); - assertThat(refused.status()).isEqualTo(ConnectionResult.Status.FAILED); - assertThat(refused.message()).contains("Permission denied"); - } - assertThat(provider.testConnection(config("http://127.0.0.1:1")).message()).contains("cannot be reached"); - assertThat(provider.testConnection(config("not an address")).status()).isEqualTo(ConnectionResult.Status.FAILED); + Files.createDirectories(root.resolve("user/alice")); + Files.createDirectories(root.resolve("user/bob")); + Files.writeString(root.resolve("user/notes.txt"), "x"); + ServiceConfig local = config("file:///"); + + assertThat(provider.testConnection(local)).isEqualTo(ConnectionResult.succeeded()); + String user = root.resolve("user").toString(); + assertThat(lookup(local, user + "/")).containsExactly(user + "/alice", user + "/bob", user + "/notes.txt"); + assertThat(lookup(local, user + "/a")).containsExactly(user + "/alice"); + assertThat(lookup(local, root + "/missing/x")).isEmpty(); + assertThat(provider.lookup(new LookupRequest(local, HdfsProvider.PATH, user + "/", Map.of(), 1))).hasSize(1); + assertThat(provider.lookup(new LookupRequest(local, "other", "", Map.of(), 10))).isEmpty(); } @Test - void looksUpPathsBelowWhatWasTyped() throws IOException + void speaksWebHdfsAsTheUser() throws IOException { try (FakeWebHdfs namenode = new FakeWebHdfs(false)) { - ServiceConfig config = config(namenode.uri().toString(), "username", " "); + ServiceConfig config = config("webhdfs://127.0.0.1:" + namenode.uri().getPort()); + assertThat(provider.testConnection(config)).isEqualTo(ConnectionResult.succeeded()); assertThat(lookup(config, "")).containsExactly("/tmp", "/user"); - assertThat(lookup(config, "/user/")).containsExactly("/user/alice", "/user/bob", "/user/notes.txt"); - assertThat(lookup(config, "user/a")).containsExactly("/user/alice"); - assertThat(lookup(config, "/user/n")).containsExactly("/user/notes.txt"); + assertThat(lookup(config, "user/")).containsExactly("/user/alice", "/user/bob", "/user/notes.txt"); assertThat(lookup(config, "/missing/x")).isEmpty(); - assertThat(provider.lookup(new LookupRequest(config, "other", "", Map.of(), 10))).isEmpty(); - assertThat(provider.lookup(new LookupRequest(config, HdfsProvider.PATH, "/user/", Map.of(), 1))).hasSize(1); - assertThat(namenode.requests).allMatch(request -> request.contains("user.name=hdfs")); + assertThat(namenode.requests).isNotEmpty().allMatch(request -> request.contains("user.name=hdfs")); + ConnectionResult refused = provider.testConnection(config("webhdfs://127.0.0.1:" + namenode.uri().getPort(), "username", "nobody")); + assertThat(refused.status()).isEqualTo(ConnectionResult.Status.FAILED); + assertThat(refused.message()).contains("Permission denied"); } - assertThatThrownBy(() -> lookup(config("http://127.0.0.1:1"), "/")).isInstanceOf(UncheckedIOException.class); + } + + @Test + void reportsClustersThatCannotBeReached() + { + ConnectionResult gone = provider.testConnection(config("hdfs://127.0.0.1:1")); + assertThat(gone.status()).isEqualTo(ConnectionResult.Status.FAILED); + assertThat(gone.message()).isNotBlank().doesNotContain("\n"); + assertThatThrownBy(() -> lookup(config("hdfs://127.0.0.1:1"), "/")).isInstanceOf(UncheckedIOException.class); + assertThat(provider.testConnection(config("hdfs://nn:8020", "hadoop.config", "broken")).status()) + .isEqualTo(ConnectionResult.Status.FAILED); + ConnectionResult noKeytab = provider.testConnection(config("hdfs://127.0.0.1:1", "hadoop.security.authentication", "kerberos", + "keytab", root.resolve("missing.keytab").toString())); + assertThat(noKeytab.status()).isEqualTo(ConnectionResult.Status.FAILED); + ConnectionResult noPassword = provider.testConnection(config("hdfs://127.0.0.1:1", "hadoop.security.authentication", "kerberos")); + assertThat(noPassword.message()).contains("password or a keytab"); } } diff --git a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/KerberosLoginTest.java b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/KerberosLoginTest.java new file mode 100644 index 00000000..96f40a7f --- /dev/null +++ b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/KerberosLoginTest.java @@ -0,0 +1,110 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.hdfs; + +import org.apache.kerby.kerberos.kerb.server.SimpleKdcServer; +import org.devlive.grantforge.plugin.api.ConnectionResult; +import org.devlive.grantforge.plugin.api.LookupRequest; +import org.devlive.grantforge.plugin.api.ServiceConfig; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.IOException; +import java.net.ServerSocket; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Signs the lookup user in against a real KDC, with a keytab and with a password, as a Kerberos cluster needs. A local + * file system stands in for the cluster, which Hadoop lists the same way once the user is signed in. + */ +class KerberosLoginTest +{ + private static final String REALM = "EXAMPLE.COM"; + private static final String PASSWORD_PRINCIPAL = "grantforge@" + REALM; + private static final String KEYTAB_PRINCIPAL = "lookup@" + REALM; + private static final String SECRET = "kerberos-secret"; + + @TempDir + static Path work; + + private static SimpleKdcServer kdc; + private static File keytab; + + private final HdfsProvider provider = new HdfsProvider(); + + @BeforeAll + static void startKdc() throws Exception + { + int port; + try (ServerSocket socket = new ServerSocket(0)) { + port = socket.getLocalPort(); + } + kdc = new SimpleKdcServer(); + kdc.setWorkDir(work.toFile()); + kdc.setKdcRealm(REALM); + kdc.setKdcHost("localhost"); + kdc.setAllowUdp(false); + kdc.setKdcTcpPort(port); + kdc.init(); + kdc.start(); + // The export takes every principal there is, so the password principal comes after it. + keytab = work.resolve("lookup.keytab").toFile(); + kdc.createAndExportPrincipals(keytab, KEYTAB_PRINCIPAL); + kdc.createPrincipal(PASSWORD_PRINCIPAL, SECRET); + // This JVM is the test class's own (surefire does not reuse forks here), so the setting cannot leak. + System.setProperty("java.security.krb5.conf", work.resolve("krb5.conf").toString()); + } + + @AfterAll + static void stopKdc() throws Exception + { + kdc.stop(); + } + + private static ServiceConfig kerberos(String user, String... more) + { + Map values = new java.util.HashMap<>(Map.of("fs.default.name", "file:///", "username", user, + "hadoop.security.authentication", "kerberos")); + for (int index = 0; index < more.length; index += 2) { + values.put(more[index], more[index + 1]); + } + return new ServiceConfig("secure-lake", values); + } + + @Test + void signsInWithAKeytab() throws IOException + { + Files.createDirectories(work.resolve("data/sales")); + ServiceConfig config = kerberos(KEYTAB_PRINCIPAL, "keytab", keytab.getPath()); + + assertThat(provider.testConnection(config)).isEqualTo(ConnectionResult.succeeded()); + String data = work.resolve("data").toString(); + assertThat(provider.lookup(new LookupRequest(config, "path", data + "/", Map.of(), 10))).containsExactly(data + "/sales"); + } + + @Test + void signsInWithAPassword() + { + assertThat(provider.testConnection(kerberos(PASSWORD_PRINCIPAL, "password", SECRET))).isEqualTo(ConnectionResult.succeeded()); + } + + @Test + void reportsRefusedCredentials() + { + ConnectionResult wrong = provider.testConnection(kerberos(PASSWORD_PRINCIPAL, "password", "not the secret")); + assertThat(wrong.status()).isEqualTo(ConnectionResult.Status.FAILED); + assertThat(wrong.message()).contains("Kerberos refused " + PASSWORD_PRINCIPAL); + ConnectionResult otherKeytab = provider.testConnection(kerberos(PASSWORD_PRINCIPAL, "keytab", keytab.getPath())); + assertThat(otherKeytab.status()).isEqualTo(ConnectionResult.Status.FAILED); + } +} diff --git a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/WebHdfsTest.java b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/WebHdfsTest.java deleted file mode 100644 index 84318e6e..00000000 --- a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/WebHdfsTest.java +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright (c) 2026 devlive-community/grantforge -// -// Licensed under the MIT License. See the LICENSE file in the -// project root for full license text. - -package org.devlive.grantforge.hdfs; - -import com.sun.net.httpserver.HttpServer; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.io.OutputStream; -import java.net.InetSocketAddress; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -class WebHdfsTest -{ - private static final Duration TIMEOUT = Duration.ofSeconds(5); - - @Test - void parsesCommaSeparatedAddresses() - { - assertThat(WebHdfs.addresses(" http://nn1:9870/ , https://nn2:9871,")) - .containsExactly(URI.create("http://nn1:9870"), URI.create("https://nn2:9871")); - assertThatIllegalArgumentException().isThrownBy(() -> WebHdfs.addresses(" , ")).withMessageContaining("no address"); - assertThatIllegalArgumentException().isThrownBy(() -> WebHdfs.addresses("hdfs://nn1:8020")).withMessageContaining("http"); - assertThatIllegalArgumentException().isThrownBy(() -> WebHdfs.addresses("http://")).withMessageContaining("http"); - assertThatIllegalArgumentException().isThrownBy(() -> WebHdfs.addresses("http://bad host")).withMessageContaining("not an address"); - assertThatIllegalArgumentException().isThrownBy(() -> new WebHdfs(List.of(), "hdfs", TIMEOUT)).withMessageContaining("no WebHDFS"); - } - - @Test - void encodesPathSegments() - { - assertThat(WebHdfs.encode("/")).isEqualTo("/"); - assertThat(WebHdfs.encode("/user/a b/x+y")).isEqualTo("/user/a%20b/x%2By"); - assertThat(WebHdfs.encode("data/")).isEqualTo("/data/"); - } - - @Test - void readsStatusesAndListingsAsTheUser() throws IOException - { - try (FakeWebHdfs namenode = new FakeWebHdfs(false)) { - WebHdfs client = new WebHdfs(List.of(namenode.uri()), "alice", TIMEOUT); - - assertThat(client.status("/")).isEqualTo(new WebHdfs.Entry("", true)); - assertThat(client.status("/user/notes.txt")).isEqualTo(new WebHdfs.Entry("", false)); - assertThat(client.status("/missing")).isNull(); - assertThat(client.list("/user")).contains(new WebHdfs.Entry("alice", true), new WebHdfs.Entry("notes.txt", false)); - assertThat(client.list("/missing")).isEmpty(); - assertThat(namenode.requests).allMatch(request -> request.contains("user.name=alice")); - } - } - - @Test - void asksTheActiveNameNodeOfAPair() throws IOException - { - try (FakeWebHdfs standby = new FakeWebHdfs(true); FakeWebHdfs active = new FakeWebHdfs(false)) { - WebHdfs client = new WebHdfs(List.of(standby.uri(), active.uri()), "hdfs", TIMEOUT); - - assertThat(client.list("/")).hasSize(2); - assertThat(standby.requests).hasSize(1); - WebHdfs onlyStandby = new WebHdfs(List.of(standby.uri()), "hdfs", TIMEOUT); - assertThatThrownBy(() -> onlyStandby.list("/")).isInstanceOf(WebHdfs.StandbyException.class).hasMessageContaining("standby"); - } - } - - @Test - void reportsRefusalsAndUnreachableNameNodes() throws IOException - { - try (FakeWebHdfs namenode = new FakeWebHdfs(false)) { - WebHdfs nobody = new WebHdfs(List.of(namenode.uri()), "nobody", TIMEOUT); - assertThatThrownBy(() -> nobody.list("/")).isInstanceOf(WebHdfs.Refused.class).hasMessageContaining("403") - .hasMessageContaining("AccessControlException").hasMessageContaining("Permission denied"); - // A refusal is final: the next address is not asked. - WebHdfs pair = new WebHdfs(List.of(namenode.uri(), URI.create("http://127.0.0.1:1")), "nobody", TIMEOUT); - assertThatThrownBy(() -> pair.list("/")).isInstanceOf(WebHdfs.Refused.class); - } - WebHdfs gone = new WebHdfs(List.of(URI.create("http://127.0.0.1:1")), "hdfs", Duration.ofSeconds(2)); - assertThatThrownBy(() -> gone.list("/")).isInstanceOf(IOException.class).hasMessageContaining("cannot be reached"); - } - - @Test - void tellsWhenTheAddressIsNotWebHdfs() throws IOException - { - HttpServer web = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); - web.createContext("/", exchange -> { - boolean ok = exchange.getRequestURI().getQuery().contains("GETFILESTATUS"); - byte[] body = "hello".getBytes(StandardCharsets.UTF_8); - exchange.sendResponseHeaders(ok ? 200 : 404, body.length); - try (OutputStream out = exchange.getResponseBody()) { - out.write(body); - } - }); - web.start(); - try { - WebHdfs client = new WebHdfs(List.of(URI.create("http://127.0.0.1:" + web.getAddress().getPort())), "hdfs", TIMEOUT); - assertThatThrownBy(() -> client.status("/")).isInstanceOf(IOException.class).hasMessageContaining("JSON"); - assertThatThrownBy(() -> client.list("/")).isInstanceOf(WebHdfs.Refused.class).hasMessageContaining("WebHDFS address"); - } - finally { - web.stop(0); - } - } -} diff --git a/pom.xml b/pom.xml index 9b815a16..49a9e553 100644 --- a/pom.xml +++ b/pom.xml @@ -74,6 +74,10 @@ 1.37 3.1.1 1.86 + + 3.5.0 + + 2.1.2 h2 @@ -255,6 +260,21 @@ bcprov-jdk18on ${bouncycastle.version} + + org.apache.hadoop + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime + ${hadoop.version} + + + org.apache.kerby + kerb-simplekdc + ${kerby.version} + @@ -289,6 +309,11 @@ maven-assembly-plugin ${plugin.assembly.version} + + org.apache.maven.plugins + maven-dependency-plugin + ${plugin.dependency.version} + org.apache.maven.plugins maven-pmd-plugin From 92aabb17e7134c4d9107216f4dc17002d451b335 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 02:11:13 -0400 Subject: [PATCH 09/22] style(hdfs): clear the Error Prone warnings of the HDFS plugin --- .../main/java/org/devlive/grantforge/hdfs/HadoopClient.java | 2 +- .../src/test/java/org/devlive/grantforge/hdfs/FakeWebHdfs.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HadoopClient.java b/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HadoopClient.java index c7eff9a0..52aede5a 100644 --- a/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HadoopClient.java +++ b/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HadoopClient.java @@ -94,7 +94,7 @@ static Map properties(@Nullable String text) return properties; } int number = 0; - for (String line : text.split("\\R")) { + for (String line : text.lines().toList()) { number++; String trimmed = line.strip(); if (trimmed.isEmpty() || trimmed.startsWith("#")) { diff --git a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/FakeWebHdfs.java b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/FakeWebHdfs.java index 943bf2eb..77aa59ff 100644 --- a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/FakeWebHdfs.java +++ b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/FakeWebHdfs.java @@ -10,6 +10,7 @@ import java.io.IOException; import java.io.OutputStream; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.URI; import java.net.URLDecoder; @@ -40,7 +41,7 @@ final class FakeWebHdfs FakeWebHdfs(boolean standby) throws IOException { this.standby = standby; - server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); server.createContext("/webhdfs/v1", this::handle); server.createContext("/", exchange -> respond(exchange, 404, "Not Found")); server.start(); From 3796c313fca9a5d7d0352d565d4f11fbe82171da Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 05:32:59 -0400 Subject: [PATCH 10/22] fix(plugin-host): load only installable plugin modules from the sources The example plugin, a test fixture, showed up as a service type in IDE runs and was picked by mistake. A module now loads as a plugin only when its build copied its libraries to target/plugin-lib, as installable plugins do. --- .../grantforge/plugin/host/PluginPackage.java | 20 +++++++++++++------ .../plugin/host/PluginPackageTest.java | 9 ++++++--- docs/content/architecture/plugins.md | 2 +- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/core/grantforge-plugin-host/src/main/java/org/devlive/grantforge/plugin/host/PluginPackage.java b/core/grantforge-plugin-host/src/main/java/org/devlive/grantforge/plugin/host/PluginPackage.java index 35e4da25..9d5d7c86 100644 --- a/core/grantforge-plugin-host/src/main/java/org/devlive/grantforge/plugin/host/PluginPackage.java +++ b/core/grantforge-plugin-host/src/main/java/org/devlive/grantforge/plugin/host/PluginPackage.java @@ -40,7 +40,8 @@ * *

A plugin's Maven module counts too once it is built, so a server started from the sources loads the plugins of * the repository (D-89): its classes come from {@code target/classes} and its dependencies from - * {@code target/plugin-lib}, which the plugin's build copies there. + * {@code target/plugin-lib}, which the build of a plugin meant to be installed copies there. A module whose build + * does not, such as the example plugin the tests install, is no plugin of the server. * * @param location the file or directory name in the plugins directory * @param descriptor what the plugin says about itself @@ -70,7 +71,7 @@ public record PluginPackage(String location, PluginDescriptor descriptor, List urls = new ArrayList<>(); urls.add(classes.toUri().toURL()); Path lib = module.resolve(MODULE_LIB); - if (Files.isDirectory(lib)) { - urls.addAll(jars(lib)); + if (!Files.isDirectory(lib)) { + throw new IllegalArgumentException(location + " has no " + MODULE_LIB + "; build it once: ./mvnw -pl plugins/" + location + + " -am install -DskipTests"); } + urls.addAll(jars(lib)); return new PluginPackage(location, descriptor, urls); } diff --git a/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/PluginPackageTest.java b/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/PluginPackageTest.java index 5b645d79..f3d4351e 100644 --- a/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/PluginPackageTest.java +++ b/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/PluginPackageTest.java @@ -37,11 +37,14 @@ void candidatesAreJarsZipsAndDirectoriesButNotHiddenOnes() assertThat(PluginPackage.candidate(source)).isFalse(); Files.writeString(source.resolve(PluginDescriptor.FILE_NAME), "id: example"); assertThat(PluginPackage.candidate(source)).isTrue(); - // A module counts once its build put the descriptor into target/classes. + // A module counts once its build put the descriptor into target/classes and copied its libraries; the example + // plugin, whose build copies none, stays a test fixture. Path module = Files.createDirectories(root.resolve("grantforge-plugin-hdfs")); Files.writeString(module.resolve("pom.xml"), ""); assertThat(PluginPackage.candidate(module)).isFalse(); Files.writeString(Files.createDirectories(module.resolve(PluginPackage.MODULE_CLASSES)).resolve(PluginDescriptor.FILE_NAME), "id: hdfs"); + assertThat(PluginPackage.candidate(module)).isFalse(); + Files.createDirectories(module.resolve(PluginPackage.MODULE_LIB)); assertThat(PluginPackage.candidate(module)).isTrue(); } @@ -60,8 +63,8 @@ void readsABuiltModuleWithTheDependenciesItsBuildCopied() providers: org.example.HdfsProvider """); - assertThat(PluginPackage.read(module, root.resolve(PluginPackage.WORK)).urls()).extracting(url -> url.getPath()) - .singleElement().asString().endsWith("target/classes/"); + assertThatThrownBy(() -> PluginPackage.read(module, root.resolve(PluginPackage.WORK))).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("./mvnw -pl plugins/grantforge-plugin-hdfs -am install -DskipTests"); Path lib = Files.createDirectories(module.resolve(PluginPackage.MODULE_LIB)); Files.writeString(lib.resolve("hadoop-client-api.jar"), ""); Files.writeString(lib.resolve("notes.txt"), ""); diff --git a/docs/content/architecture/plugins.md b/docs/content/architecture/plugins.md index 2873fe9f..f5ff0d14 100644 --- a/docs/content/architecture/plugins.md +++ b/docs/content/architecture/plugins.md @@ -121,7 +121,7 @@ providers: ## 从源码启动时 -在 IDE 里直接启动 `org.devlive.grantforge.server.GrantForge` 时，服务端的类来自各模块的 `target/classes`，此时如果没有配置 `grantforge.plugins.directory`、工作目录下也没有 `plugins` 目录，就使用仓库的 `plugins/` 目录：其中构建过的插件模块（`target/classes` 里有描述符）直接作为插件加载，类来自 `target/classes`，依赖来自 `target/plugin-lib`。修改插件代码后由 IDE 重新编译，在控制台“插件”页重新扫描即可生效。插件模块第一次使用前，用 Maven 构建一次以复制依赖： +在 IDE 里直接启动 `org.devlive.grantforge.server.GrantForge` 时，服务端的类来自各模块的 `target/classes`，此时如果没有配置 `grantforge.plugins.directory`、工作目录下也没有 `plugins` 目录，就使用仓库的 `plugins/` 目录：其中构建过的插件模块（`target/classes` 里有描述符，且构建生成了 `target/plugin-lib`）直接作为插件加载，类来自 `target/classes`，依赖来自 `target/plugin-lib`；不生成 `plugin-lib` 的模块（如测试用的示例插件）不会加载。修改插件代码后由 IDE 重新编译，在控制台“插件”页重新扫描即可生效。插件模块第一次使用前，用 Maven 构建一次以复制依赖： ```bash ./mvnw -pl plugins/grantforge-plugin-hdfs -am install -DskipTests From 1437ec52a97432aef9354436930a80ff069f72b5 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 05:32:59 -0400 Subject: [PATCH 11/22] build: unpack the bundled HDFS plugin when packaging the server The server declared the plugin's zip as a dependency, which exists only from the package phase on, so every CI job that stops at test failed to resolve it. The root pom now builds the plugin before the server, which unpacks the zip at prepare-package for the release. --- configure/assembly/server.xml | 15 +++++--------- core/grantforge-server/pom.xml | 36 +++++++++++++++++++++++++--------- pom.xml | 3 ++- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/configure/assembly/server.xml b/configure/assembly/server.xml index 366a42bc..88579763 100644 --- a/configure/assembly/server.xml +++ b/configure/assembly/server.xml @@ -20,19 +20,14 @@ lib false - - - false - provided - - org.devlive.grantforge:grantforge-plugin-hdfs:zip:plugin - - plugins/hdfs - true - + + + target/bundled-plugins + plugins + ${project.parent.basedir}/script/bin bin diff --git a/core/grantforge-server/pom.xml b/core/grantforge-server/pom.xml index 42f9ad02..eb565f14 100644 --- a/core/grantforge-server/pom.xml +++ b/core/grantforge-server/pom.xml @@ -65,15 +65,6 @@ org.springframework.boot spring-boot-starter-liquibase - - - org.devlive.grantforge - grantforge-plugin-hdfs - ${project.version} - plugin - zip - provided - org.springframework.boot @@ -198,6 +189,33 @@ + + + org.apache.maven.plugins + maven-dependency-plugin + + + bundled-plugins + prepare-package + + unpack + + + + + org.devlive.grantforge + grantforge-plugin-hdfs + ${project.version} + plugin + zip + ${project.build.directory}/bundled-plugins/hdfs + + + + + + org.apache.maven.plugins maven-assembly-plugin diff --git a/pom.xml b/pom.xml index 49a9e553..b05a11b7 100644 --- a/pom.xml +++ b/pom.xml @@ -31,10 +31,11 @@ core/grantforge-plugin-host core/grantforge-service core/grantforge-oauth + + plugins/grantforge-plugin-hdfs core/grantforge-server sdk/grantforge-spring-boot-starter plugins/grantforge-plugin-example - plugins/grantforge-plugin-hdfs From 607b6a8b0b8efdc82a8e16598c83bc3584a87a78 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 05:32:59 -0400 Subject: [PATCH 12/22] ci: accept the Bouncy Castle licence in dependency review The Bouncy Castle Licence is the MIT license under its own name; the review reported it as non-standard. --- .github/dependency-review-config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/dependency-review-config.yml b/.github/dependency-review-config.yml index c6045110..94211b79 100644 --- a/.github/dependency-review-config.yml +++ b/.github/dependency-review-config.yml @@ -38,4 +38,6 @@ allow-dependencies-licenses: # BSD-2-Clause together with the equally permissive BSD-2-Clause-Views. - pkg:npm/robust-predicates - pkg:npm/uri-js + # The Bouncy Castle Licence is the MIT license under its own name, which the review does not recognise. + - pkg:maven/org.bouncycastle/bcprov-jdk18on comment-summary-in-pr: on-failure From 455193b79900f2bc3225ad80b9e26d6d47bd69de Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 05:56:57 -0400 Subject: [PATCH 13/22] ci: check commit messages after a force push A push reports the branch's previous tip as the range start; after a force push that commit is gone from the clone, and the check crashed on an invalid range. It now checks the branch from where it left the default branch, as for a pull request, or the head alone without one. --- .github/workflows/ci.yml | 6 +++- script/ci/check_commit_messages.py | 28 ++++++++++++++++++- script/ci/tests/test_check_commit_messages.py | 14 ++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a43cc9cf..5865a1a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,7 +69,11 @@ jobs: with: python-version: '3.12' - name: Conventional commit messages - run: python3 script/ci/check_commit_messages.py --base "${{ github.event.pull_request.base.sha || github.event.before }}" --head "${{ github.event.pull_request.head.sha || github.sha }}" + run: >- + python3 script/ci/check_commit_messages.py + --base "${{ github.event.pull_request.base.sha || github.event.before }}" + --head "${{ github.event.pull_request.head.sha || github.sha }}" + --fallback "origin/${{ github.event.repository.default_branch }}" ci-scripts: name: CI script unit tests diff --git a/script/ci/check_commit_messages.py b/script/ci/check_commit_messages.py index fc74f8eb..c441b5ee 100755 --- a/script/ci/check_commit_messages.py +++ b/script/ci/check_commit_messages.py @@ -126,6 +126,30 @@ def validate_message(message: str) -> List[str]: return errors +def _is_commit(root: Path, revision: str) -> bool: + return subprocess.run(["git", "cat-file", "-e", f"{revision}^{{commit}}"], cwd=root, check=False, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0 + + +def resolve_base(root: Path, base: Optional[str], head: str, fallback: Optional[str]) -> Optional[str]: + """Return the range start to use, or None to check ``head`` alone. + + A push reports the branch's previous tip as ``base``. After a force push that commit is gone from the clone; + the branch is then checked from where it left ``fallback`` (the default branch), as a pull request would be. + """ + if not base or _ZERO_SHA.match(base): + return None + if _is_commit(root, base): + return base + if fallback and _is_commit(root, fallback): + result = subprocess.run(["git", "merge-base", fallback, head], cwd=root, check=False, stdout=subprocess.PIPE) + if result.returncode == 0: + print(f"{base[:10]} is not in this clone (a force push?); checking from {fallback}") + return result.stdout.decode("utf-8").strip() + print(f"{base[:10]} is not in this clone; checking {head} alone") + return None + + def commits_in_range(root: Path, base: Optional[str], head: str) -> List[Commit]: """Return commits reachable from ``head`` but not ``base``; only ``head`` when base is unusable.""" # A new-branch push reports an all-zero 'before' SHA: there is no range, so check head only. @@ -149,6 +173,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--base", default="", help="exclusive range start (PR base or push 'before')") parser.add_argument("--head", default="HEAD", help="inclusive range end") + parser.add_argument("--fallback", default="", help="branch to check from when --base is gone, such as origin/dev") parser.add_argument("--file", type=Path, help="validate a single message file instead of a range") parser.add_argument("--root", type=Path, default=Path.cwd(), help="repository root") args = parser.parse_args(argv) @@ -160,7 +185,8 @@ def main(argv: Optional[Sequence[str]] = None) -> int: return 1 if errors else 0 failed = 0 - commits = commits_in_range(args.root, args.base, args.head) + base = resolve_base(args.root, args.base, args.head, args.fallback or None) + commits = commits_in_range(args.root, base, args.head) for commit in commits: if commit.parents > 1 or commit.author_email in BOT_AUTHOR_EMAILS: continue # merge commits and bot commits are generated by the platform diff --git a/script/ci/tests/test_check_commit_messages.py b/script/ci/tests/test_check_commit_messages.py index 9ca4cf67..905f6f36 100644 --- a/script/ci/tests/test_check_commit_messages.py +++ b/script/ci/tests/test_check_commit_messages.py @@ -127,6 +127,20 @@ def test_zero_base_checks_head_only(self) -> None: self.assertEqual(self._main("--base", "0" * 40, "--head", head), 0) self.assertEqual(self._main("--base", "", "--head", head), 0) + def test_a_force_pushed_base_falls_back_to_the_default_branch(self) -> None: + self._commit("legacy message that breaks every rule.") + self._git("branch", "dev") + self._git("checkout", "-q", "-b", "topic") + self._commit("feat: add topic") + gone = "1234567890" * 4 + self.assertEqual(self._main("--base", gone, "--head", "HEAD", "--fallback", "dev"), 0) + self._commit("oops") + self.assertEqual(self._main("--base", gone, "--head", "HEAD", "--fallback", "dev"), 1) + # Without a usable fallback only the head is checked. + self.assertEqual(self._main("--base", gone, "--head", "HEAD~1", "--fallback", "missing"), 0) + self.assertEqual(self._main("--base", gone, "--head", "HEAD~1"), 0) + self.assertEqual(chk.resolve_base(self.root, gone, "HEAD", "dev"), self._git("rev-parse", "dev")) + def test_merge_commits_are_skipped(self) -> None: base = self._commit("chore: start") self._git("checkout", "-q", "-b", "topic") From 1d20eac37d3d8dc475e755b90d37d573e148f557 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 07:12:27 -0400 Subject: [PATCH 14/22] feat(hdfs): bound path lookups and validate cluster settings Lookups can start from a configured directory (lookup.path) for users who may not list the root, and stop with an error above lookup.max.entries instead of scanning huge folders. Cluster URIs, authentication and the additional properties are validated as Hadoop will use them, and the whole call is serialized because UGI keeps its security settings in static state. Lookup input may be as long as a policy resource value. --- .../plugin/host/HdfsPluginTest.java | 200 ++++++++++++++++-- .../server/service/LookupRequestBody.java | 4 +- .../server/service/LookupRequestBodyTest.java | 15 ++ core/grantforge-web/src/api/openapi.json | 2 +- .../devlive/grantforge/hdfs/HadoopClient.java | 74 ++++--- .../devlive/grantforge/hdfs/HdfsProvider.java | 133 ++++++++++-- .../devlive/grantforge/hdfs/FakeWebHdfs.java | 6 +- .../grantforge/hdfs/HadoopClientTest.java | 94 ++++++++ .../grantforge/hdfs/HdfsProviderTest.java | 84 +++++++- .../grantforge/hdfs/KerberosLoginTest.java | 10 + 10 files changed, 548 insertions(+), 74 deletions(-) diff --git a/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/HdfsPluginTest.java b/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/HdfsPluginTest.java index 3a85e66c..ca6081e2 100644 --- a/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/HdfsPluginTest.java +++ b/core/grantforge-plugin-host/src/test/java/org/devlive/grantforge/plugin/host/HdfsPluginTest.java @@ -5,55 +5,55 @@ package org.devlive.grantforge.plugin.host; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; import org.devlive.grantforge.plugin.api.ConnectionResult; import org.devlive.grantforge.plugin.api.LookupRequest; +import org.devlive.grantforge.plugin.api.PluginDescriptor; import org.devlive.grantforge.plugin.api.ServiceConfig; +import org.devlive.grantforge.plugin.api.model.MatcherType; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.Stream; import static java.util.Objects.requireNonNull; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** - * Loads the HDFS plugin the way a server started from the sources does (D-89): straight from its built module in the - * repository's plugins folder, with Hadoop's client from the module's plugin-lib, in a class loader that sees none of - * the server's libraries; then lists a folder through Hadoop's file system. The module is built first, as this one - * depends on it. + * Loads the HDFS plugin from the sources (D-89) and from a release's directory layout, with Hadoop's client in an + * isolated class loader, then connects and looks up paths through its local and WebHDFS file systems. The module is + * built first, as this one depends on it. */ class HdfsPluginTest { @TempDir private Path files; + @TempDir + private Path installed; + @Test void loadsTheBuiltModuleWithHadoopsClient() throws Exception { Path plugins = requireNonNull(PluginDirectory.sourceTree(PluginDirectory.codeSource(HdfsPluginTest.class)), "no repository"); Files.createDirectories(files.resolve("data/sales")); - PluginSwitches on = new PluginSwitches() - { - @Override - public boolean enabled(String pluginId) - { - return true; - } - - @Override - public void set(String pluginId, boolean enabled) - { - // Always on. - } - }; // A local file system stands in for a cluster: the same Hadoop client code lists it. ServiceConfig config = new ServiceConfig("lake", Map.of("fs.default.name", "file:///", "username", "hdfs")); try (PluginCalls calls = new PluginCalls(Duration.ofSeconds(30)); - PluginRegistry registry = new PluginRegistry(plugins, on, calls, HdfsPluginTest.class.getClassLoader())) { + PluginRegistry registry = new PluginRegistry(plugins, enabled(), calls, HdfsPluginTest.class.getClassLoader())) { registry.scan(); InstalledPlugin hdfs = registry.plugins().stream().filter(plugin -> plugin.id().equals("hdfs")).findFirst().orElseThrow(); assertThat(hdfs.status()).as(String.valueOf(hdfs.problem())).isEqualTo(PluginStatus.ACTIVE); @@ -68,4 +68,166 @@ public void set(String pluginId, boolean enabled) assertThat(found).containsExactly(data + "/sales"); } } + + @Test + void loadsAReleaseDirectoryAndUsesWebHdfsWithoutTheServersLibraries() throws Exception + { + installReleaseDirectory(); + try (WebHdfs namenode = new WebHdfs(); + PluginCalls calls = new PluginCalls(Duration.ofSeconds(30)); + PluginRegistry registry = new PluginRegistry(installed, enabled(), calls, HdfsPluginTest.class.getClassLoader())) { + registry.scan(); + InstalledPlugin hdfs = registry.plugins().get(0); + assertThat(hdfs.status()).as(String.valueOf(hdfs.problem())).isEqualTo(PluginStatus.ACTIVE); + assertThat(hdfs.location()).isEqualTo("hdfs"); + assertThat(registry.serviceType("hdfs")).hasValueSatisfying(type -> { + assertThat(type.resources()).singleElement().satisfies(path -> { + assertThat(path.name()).isEqualTo("path"); + assertThat(path.matcher()).isEqualTo(MatcherType.PATH); + assertThat(path.lookupSupported()).isTrue(); + assertThat(path.recursiveSupported()).isTrue(); + }); + assertThat(type.accessTypes()).extracting(access -> access.name()).contains("read", "write", "execute"); + }); + ServiceConfig allowed = namenode.config("hdfs"); + ConnectionResult connected = registry.call("hdfs", provider -> provider.testConnection(allowed)); + assertThat(connected.status()).as(String.valueOf(connected.message())).isEqualTo(ConnectionResult.Status.SUCCEEDED); + assertThat(lookup(registry, allowed, "/data/s", 2)).containsExactly("/data/sales", "/data/support"); + assertThat(lookup(registry, allowed, "/data/销", 20)).containsExactly("/data/销售"); + assertThat(lookup(registry, allowed, "/missing/", 20)).isEmpty(); + + ServiceConfig refused = namenode.config("nobody"); + ConnectionResult failed = registry.call("hdfs", provider -> provider.testConnection(refused)); + assertThat(failed.status()).isEqualTo(ConnectionResult.Status.FAILED); + assertThat(failed.message()).contains("Permission denied"); + assertThatThrownBy(() -> lookup(registry, refused, "/data/", 20)) + .isInstanceOf(PluginCallException.class).hasMessageContaining("Permission denied"); + assertThat(registry.call("hdfs", provider -> provider.testConnection(allowed)).status()).isEqualTo(ConnectionResult.Status.SUCCEEDED); + assertThat(registry.plugins().get(0).status()).isEqualTo(PluginStatus.ACTIVE); + assertThat(namenode.requests).anyMatch(query -> query.contains("user.name=hdfs")) + .anyMatch(query -> query.contains("user.name=nobody")); + } + } + + private static List lookup(PluginRegistry registry, ServiceConfig config, String text, int limit) + { + return registry.call("hdfs", provider -> provider.lookup(new LookupRequest(config, "path", text, Map.of(), limit))); + } + + private void installReleaseDirectory() throws IOException + { + Path plugins = requireNonNull(PluginDirectory.sourceTree(PluginDirectory.codeSource(HdfsPluginTest.class)), "no repository"); + Path built = plugins.resolve("grantforge-plugin-hdfs/target"); + Path release = Files.createDirectories(installed.resolve("hdfs")); + Files.copy(built.resolve("classes").resolve(PluginDescriptor.FILE_NAME), release.resolve(PluginDescriptor.FILE_NAME)); + copyTree(built.resolve("classes"), release.resolve("classes")); + copyTree(built.resolve("plugin-lib"), release.resolve("lib")); + } + + private static void copyTree(Path source, Path destination) throws IOException + { + try (Stream files = Files.walk(source)) { + for (Path file : files.filter(Files::isRegularFile).toList()) { + Path target = destination.resolve(source.relativize(file)); + Files.createDirectories(target.getParent()); + Files.copy(file, target); + } + } + } + + private static PluginSwitches enabled() + { + return new PluginSwitches() + { + @Override + public boolean enabled(String pluginId) + { + return true; + } + + @Override + public void set(String pluginId, boolean enabled) + { + // Always on. + } + }; + } + + /** A protocol fixture; the plugin must load its own Hadoop and JSON libraries to read it. */ + private static final class WebHdfs + implements AutoCloseable + { + private final HttpServer server; + private final List requests = new CopyOnWriteArrayList<>(); + + private WebHdfs() throws IOException + { + server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + server.createContext("/webhdfs/v1", this::handle); + server.start(); + } + + private ServiceConfig config(String user) + { + return new ServiceConfig("lake", Map.of("fs.default.name", "webhdfs://127.0.0.1:" + server.getAddress().getPort(), + "username", user)); + } + + private void handle(HttpExchange exchange) throws IOException + { + String query = requireNonNull(exchange.getRequestURI().getRawQuery()); + requests.add(query); + if (query.contains("user.name=nobody")) { + respond(exchange, 403, remote("AccessControlException", "org.apache.hadoop.security.AccessControlException", "Permission denied")); + return; + } + String path = exchange.getRequestURI().getPath().substring("/webhdfs/v1".length()); + boolean root = path.isEmpty() || "/".equals(path); + boolean data = "/data".equals(path) || "/data/".equals(path); + if (!root && !data) { + respond(exchange, 404, remote("FileNotFoundException", "java.io.FileNotFoundException", "File does not exist")); + return; + } + if (query.contains("op=GETFILESTATUS")) { + respond(exchange, 200, "{\"FileStatus\":" + status("", "DIRECTORY") + "}"); + return; + } + String children = root ? status("data", "DIRECTORY") : status("summary.csv", "FILE") + "," + + status("support", "DIRECTORY") + "," + status("sales", "DIRECTORY") + "," + status("销售", "DIRECTORY"); + String statuses = "{\"FileStatuses\":{\"FileStatus\":[" + children + "]}}"; + if (query.contains("op=LISTSTATUS_BATCH")) { + respond(exchange, 200, "{\"DirectoryListing\":{\"partialListing\":" + statuses + ",\"remainingEntries\":0}}"); + } + else { + respond(exchange, 200, statuses); + } + } + + private static String remote(String type, String className, String message) + { + return "{\"RemoteException\":{\"exception\":\"" + type + "\",\"javaClassName\":\"" + className + "\",\"message\":\"" + message + "\"}}"; + } + + private static String status(String name, String type) + { + return "{\"pathSuffix\":\"" + name + "\",\"type\":\"" + type + "\",\"length\":0,\"owner\":\"hdfs\",\"group\":\"supergroup\"," + + "\"permission\":\"755\",\"accessTime\":0,\"modificationTime\":0,\"blockSize\":134217728,\"replication\":0," + + "\"fileId\":16386,\"childrenNum\":0,\"storagePolicy\":0}"; + } + + private static void respond(HttpExchange exchange, int status, String body) throws IOException + { + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(status, bytes.length); + try (OutputStream output = exchange.getResponseBody()) { + output.write(bytes); + } + } + + @Override + public void close() + { + server.stop(0); + } + } } diff --git a/core/grantforge-server/src/main/java/org/devlive/grantforge/server/service/LookupRequestBody.java b/core/grantforge-server/src/main/java/org/devlive/grantforge/server/service/LookupRequestBody.java index d4b30c4b..ae4e78ea 100644 --- a/core/grantforge-server/src/main/java/org/devlive/grantforge/server/service/LookupRequestBody.java +++ b/core/grantforge-server/src/main/java/org/devlive/grantforge/server/service/LookupRequestBody.java @@ -20,13 +20,13 @@ * What to look up in a service. * * @param resource the resource level whose values are wanted - * @param userInput what the user typed so far + * @param userInput what the user typed so far; at most 1024 characters, like a policy resource value * @param context values chosen for other levels; none when left out * @param limit the most values wanted; 20 unless given, at most 100 */ public record LookupRequestBody( @NotBlank @Size(max = 64) @Nullable String resource, - @Size(max = 256) @Nullable String userInput, + @Size(max = 1024) @Nullable String userInput, @Size(max = 20) Map> context, @Min(1) @Max(100) @Nullable Integer limit) { diff --git a/core/grantforge-server/src/test/java/org/devlive/grantforge/server/service/LookupRequestBodyTest.java b/core/grantforge-server/src/test/java/org/devlive/grantforge/server/service/LookupRequestBodyTest.java index 7c41daa5..289a04f5 100644 --- a/core/grantforge-server/src/test/java/org/devlive/grantforge/server/service/LookupRequestBodyTest.java +++ b/core/grantforge-server/src/test/java/org/devlive/grantforge/server/service/LookupRequestBodyTest.java @@ -5,6 +5,9 @@ package org.devlive.grantforge.server.service; +import jakarta.validation.Validation; +import jakarta.validation.Validator; +import jakarta.validation.ValidatorFactory; import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -18,6 +21,18 @@ @SuppressWarnings("NullAway") class LookupRequestBodyTest { + @Test + void acceptsTheSamePathLengthAsPolicyResourceValues() + { + try (ValidatorFactory factory = Validation.buildDefaultValidatorFactory()) { + Validator validator = factory.getValidator(); + String path = "/" + "a".repeat(1023); + assertThat(validator.validate(new LookupRequestBody("path", path, Map.of(), 20))).isEmpty(); + assertThat(validator.validate(new LookupRequestBody("path", path + "a", Map.of(), 20))) + .singleElement().satisfies(problem -> assertThat(problem.getPropertyPath().toString()).isEqualTo("userInput")); + } + } + @Test void leavesOutLevelsAndValuesWithoutValue() { diff --git a/core/grantforge-web/src/api/openapi.json b/core/grantforge-web/src/api/openapi.json index b0a52ae5..8dba2b89 100644 --- a/core/grantforge-web/src/api/openapi.json +++ b/core/grantforge-web/src/api/openapi.json @@ -2392,7 +2392,7 @@ "type" : "string" }, "userInput" : { - "maxLength" : 256, + "maxLength" : 1024, "minLength" : 0, "type" : "string" } diff --git a/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HadoopClient.java b/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HadoopClient.java index 52aede5a..9e572462 100644 --- a/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HadoopClient.java +++ b/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HadoopClient.java @@ -37,7 +37,7 @@ */ final class HadoopClient { - /** Hadoop keeps the Kerberos settings in static state; logins are done one at a time. */ + /** Hadoop also reads its static security settings when opening and using a file system. */ private static final ReentrantLock LOGIN = new ReentrantLock(); /** Settings that make an unreachable cluster fail within the plugin call's time limit instead of retrying. */ @@ -68,7 +68,10 @@ Configuration configuration() { Configuration hadoop = new Configuration(); FAIL_FAST.forEach(hadoop::set); - hadoop.set("fs.defaultFS", config.require(HdfsProvider.DEFAULT_FS)); + hadoop.set("fs.defaultFS", config.require(HdfsProvider.DEFAULT_FS).strip()); + // A core-site.xml on the server's class path must not silently turn a simple service into a secure one. + hadoop.set(HdfsProvider.AUTHENTICATION, HdfsProvider.SIMPLE); + hadoop.set(HdfsProvider.AUTHORIZATION, "false"); for (String name : HdfsProvider.HADOOP_SETTINGS) { String value = config.get(name); if (value != null && !value.isBlank()) { @@ -85,7 +88,7 @@ Configuration configuration() * * @param text the lines, or {@code null} * @return the properties in their order - * @throws IllegalArgumentException for a line that is not {@code key=value} + * @throws IllegalArgumentException for malformed lines, invalid property names or repeated properties */ static Map properties(@Nullable String text) { @@ -104,7 +107,13 @@ static Map properties(@Nullable String text) if (equals <= 0) { throw new IllegalArgumentException("line " + number + " is not key=value"); } - properties.put(trimmed.substring(0, equals).strip(), trimmed.substring(equals + 1).strip()); + String key = trimmed.substring(0, equals).strip(); + if (key.isEmpty() || key.chars().anyMatch(Character::isWhitespace)) { + throw new IllegalArgumentException("line " + number + " has an empty property name or whitespace in its name"); + } + if (properties.putIfAbsent(key, trimmed.substring(equals + 1).strip()) != null) { + throw new IllegalArgumentException("line " + number + " repeats property " + key); + } } return properties; } @@ -123,9 +132,15 @@ static Map properties(@Nullable String text) T run(FileSystemAction action) throws IOException { Configuration hadoop = configuration(); - UserGroupInformation user = login(hadoop); URI address = FileSystem.getDefaultUri(hadoop); + boolean locked = false; try { + // Lock the whole call: serializing just the login allows another service to change UGI's static + // authentication and auth-to-local settings before this service opens its own file system. + // Waiting is interruptible so the host's call timeout can cancel a queued lookup. + LOGIN.lockInterruptibly(); + locked = true; + UserGroupInformation user = login(hadoop); return user.doAs((PrivilegedExceptionAction) () -> { try (FileSystem files = FileSystem.newInstance(address, hadoop)) { return action.apply(files); @@ -142,20 +157,23 @@ T run(FileSystemAction action) throws IOException } throw new IOException(String.valueOf(wrapped.getCause()), wrapped); } + finally { + if (locked) { + LOGIN.unlock(); + } + } } private UserGroupInformation login(Configuration hadoop) throws IOException { String user = config.require(HdfsProvider.USER).strip(); - if (!HdfsProvider.KERBEROS.equals(config.get(HdfsProvider.AUTHENTICATION))) { - LOGIN.lock(); - try { - UserGroupInformation.setConfiguration(hadoop); - return UserGroupInformation.createRemoteUser(user); - } - finally { - LOGIN.unlock(); - } + String authentication = hadoop.getTrimmed(HdfsProvider.AUTHENTICATION, HdfsProvider.SIMPLE); + if (HdfsProvider.SIMPLE.equals(authentication)) { + UserGroupInformation.setConfiguration(hadoop); + return UserGroupInformation.createRemoteUser(user); + } + if (!HdfsProvider.KERBEROS.equals(authentication)) { + throw new IOException("unsupported Hadoop authentication type: " + authentication); } String keytab = config.get(HdfsProvider.KEYTAB); String password = config.get(HdfsProvider.PASSWORD); @@ -164,26 +182,20 @@ private UserGroupInformation login(Configuration hadoop) throws IOException if (!withKeytab && (password == null || password.isEmpty())) { throw new IOException("Kerberos needs the lookup user's password or a keytab"); } - LOGIN.lock(); try { - try { - UserGroupInformation.setConfiguration(hadoop); - } - catch (IllegalArgumentException unconfigured) { - throw new IOException("Kerberos is not set up on the GrantForge server (krb5.conf, or java.security.krb5.realm and" - + " .kdc): " + unconfigured.getMessage(), unconfigured); - } - UserGroupInformation signedIn = keytabPath != null ? UserGroupInformation.loginUserFromKeytabAndReturnUGI(user, keytabPath) - : UserGroupInformation.getUGIFromSubject(passwordLogin(user, requireNonNull(password, "password"))); - // A local file system asks for no credentials; a cluster would refuse later, so tell now. - if (!signedIn.hasKerberosCredentials() || !user.equals(signedIn.getUserName())) { - throw new IOException("Kerberos gave no credentials of " + user + (keytabPath == null ? "" : "; is it in " + keytabPath + "?")); - } - return signedIn; + UserGroupInformation.setConfiguration(hadoop); } - finally { - LOGIN.unlock(); + catch (IllegalArgumentException unconfigured) { + throw new IOException("Kerberos is not set up on the GrantForge server (krb5.conf, or java.security.krb5.realm and" + + " .kdc): " + unconfigured.getMessage(), unconfigured); + } + UserGroupInformation signedIn = keytabPath != null ? UserGroupInformation.loginUserFromKeytabAndReturnUGI(user, keytabPath) + : UserGroupInformation.getUGIFromSubject(passwordLogin(user, requireNonNull(password, "password"))); + // A local file system asks for no credentials; a cluster would refuse later, so tell now. + if (!signedIn.hasKerberosCredentials() || !user.equals(signedIn.getUserName())) { + throw new IOException("Kerberos gave no credentials of " + user + (keytabPath == null ? "" : "; is it in " + keytabPath + "?")); } + return signedIn; } /** Signs a principal in with its password through the JDK's Kerberos login module. */ diff --git a/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HdfsProvider.java b/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HdfsProvider.java index 9b81b6e3..acd62748 100644 --- a/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HdfsProvider.java +++ b/plugins/grantforge-plugin-hdfs/src/main/java/org/devlive/grantforge/hdfs/HdfsProvider.java @@ -7,6 +7,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.ipc.RemoteException; import org.devlive.grantforge.plugin.api.ConnectionResult; import org.devlive.grantforge.plugin.api.LookupRequest; @@ -26,10 +27,10 @@ import java.io.UncheckedIOException; import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; import java.util.Comparator; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; /** @@ -57,6 +58,8 @@ public final class HdfsProvider static final String SECONDARY_PRINCIPAL = "dfs.secondary.namenode.kerberos.principal"; static final String RPC_PROTECTION = "hadoop.rpc.protection"; static final String EXTRA = "hadoop.config"; + static final String LOOKUP_ROOT = "lookup.path"; + static final String LOOKUP_MAX = "lookup.max.entries"; static final String SIMPLE = "simple"; static final String KERBEROS = "kerberos"; @@ -97,7 +100,11 @@ public ServiceTypeDefinition definition() ConfigField.builder(EXTRA).label("Additional Hadoop properties").type(ConfigFieldType.TEXT) .description("One key=value per line, such as the HA nameservice: dfs.nameservices," + " dfs.ha.namenodes., dfs.namenode.rpc-address..," - + " dfs.client.failover.proxy.provider.").build()) + + " dfs.client.failover.proxy.provider.").build(), + ConfigField.builder(LOOKUP_ROOT).label("Lookup directory").type(ConfigFieldType.STRING).defaultValue("/") + .description("Absolute directory to test and browse, such as /data; lookup stays below this path").build(), + ConfigField.builder(LOOKUP_MAX).label("Maximum directory entries").type(ConfigFieldType.INTEGER).defaultValue("10000") + .description("Stop with an error above this many entries; between 1 and 100000").build()) .build(); } @@ -105,36 +112,97 @@ public ServiceTypeDefinition definition() public List validateConfig(ServiceConfig config) { List problems = new ArrayList<>(); - String address = config.get(DEFAULT_FS); - if (address != null) { - String scheme = scheme(address); - if (scheme == null || !SCHEMES.contains(scheme)) { - problems.add(new ConfigProblem(DEFAULT_FS, ConfigProblem.Reason.INVALID, "use hdfs://, webhdfs://, swebhdfs:// or viewfs://")); + Map extra; + try { + extra = HadoopClient.properties(config.get(EXTRA)); + } + catch (IllegalArgumentException invalid) { + problems.add(new ConfigProblem(EXTRA, ConfigProblem.Reason.INVALID, invalid.getMessage())); + extra = Map.of(); + } + String address = extra.getOrDefault("fs.defaultFS", extra.getOrDefault(DEFAULT_FS, config.get(DEFAULT_FS))); + if (extra.containsKey("fs.defaultFS") && extra.containsKey(DEFAULT_FS)) { + problems.add(new ConfigProblem(EXTRA, ConfigProblem.Reason.INVALID, + "fs.defaultFS and fs.default.name are aliases; configure only one")); + } + else if (address != null) { + if (!validAddress(address)) { + String field = extra.containsKey("fs.defaultFS") || extra.containsKey(DEFAULT_FS) ? EXTRA : DEFAULT_FS; + problems.add(new ConfigProblem(field, ConfigProblem.Reason.INVALID, + "use a cluster URI (hdfs://, webhdfs://, swebhdfs:// or viewfs://), without credentials, a path, query or fragment")); } } - if (KERBEROS.equals(config.get(AUTHENTICATION)) && blank(config.get(PASSWORD)) && blank(config.get(KEYTAB))) { + String authentication = extra.getOrDefault(AUTHENTICATION, config.get(AUTHENTICATION)); + if (authentication != null && !Set.of(SIMPLE, KERBEROS).contains(authentication.strip())) { + problems.add(new ConfigProblem(extra.containsKey(AUTHENTICATION) ? EXTRA : AUTHENTICATION, + ConfigProblem.Reason.INVALID, "authentication must be simple or kerberos")); + } + if (authentication != null && KERBEROS.equals(authentication.strip()) && blank(config.get(PASSWORD)) && blank(config.get(KEYTAB))) { problems.add(new ConfigProblem(PASSWORD, ConfigProblem.Reason.REQUIRED, "Kerberos needs a password or a keytab")); } try { - HadoopClient.properties(config.get(EXTRA)); + lookupRoot(config); } catch (IllegalArgumentException invalid) { - problems.add(new ConfigProblem(EXTRA, ConfigProblem.Reason.INVALID, invalid.getMessage())); + problems.add(new ConfigProblem(LOOKUP_ROOT, ConfigProblem.Reason.INVALID, invalid.getMessage())); + } + try { + scanLimit(config); + } + catch (IllegalArgumentException invalid) { + problems.add(new ConfigProblem(LOOKUP_MAX, ConfigProblem.Reason.INVALID, invalid.getMessage())); } return problems; } - private static @Nullable String scheme(String address) + private static boolean validAddress(String address) { try { - String scheme = URI.create(address.strip()).getScheme(); - return scheme == null ? null : scheme.toLowerCase(Locale.ROOT); + URI uri = URI.create(address.strip()); + String scheme = uri.getScheme(); + return scheme != null && SCHEMES.contains(scheme.toLowerCase(Locale.ROOT)) && scheme.equals(scheme.toLowerCase(Locale.ROOT)) + && !uri.isOpaque() && uri.getUserInfo() == null && uri.getQuery() == null && uri.getFragment() == null + && (uri.getPath() == null || uri.getPath().isEmpty() || "/".equals(uri.getPath())) + && ("viewfs".equals(scheme) || (uri.getHost() != null && !uri.getHost().isBlank())) + && (uri.getPort() == -1 || uri.getPort() > 0 && uri.getPort() <= 65535); } catch (IllegalArgumentException invalid) { - return null; + return false; } } + private static String lookupRoot(ServiceConfig config) + { + String root = config.get(LOOKUP_ROOT); + return normalizePath(root == null || root.isBlank() ? "/" : root.strip()); + } + + private static String normalizePath(String path) + { + if (!path.startsWith("/") || path.contains("://") || path.indexOf('\0') >= 0) { + throw new IllegalArgumentException("use an absolute file system path"); + } + List components = new ArrayList<>(); + for (String component : path.split("/")) { + if ("..".equals(component)) { + throw new IllegalArgumentException("parent path segments (..) are not allowed"); + } + if (!component.isEmpty() && !".".equals(component)) { + components.add(component); + } + } + return "/" + String.join("/", components); + } + + private static int scanLimit(ServiceConfig config) + { + long limit = config.getLong(LOOKUP_MAX, 10000); + if (limit < 1 || limit > 100000) { + throw new IllegalArgumentException("maximum directory entries must be between 1 and 100000"); + } + return (int) limit; + } + private static boolean blank(@Nullable String value) { return value == null || value.isBlank(); @@ -144,8 +212,16 @@ private static boolean blank(@Nullable String value) public ConnectionResult testConnection(ServiceConfig config) { try { - FileStatus root = new HadoopClient(config).run(files -> files.getFileStatus(new Path("/"))); - return root.isDirectory() ? ConnectionResult.succeeded() : ConnectionResult.failed("the root of the file system is not a directory"); + String directory = lookupRoot(config); + return new HadoopClient(config).run(files -> { + FileStatus root = files.getFileStatus(new Path(directory)); + if (!root.isDirectory()) { + return ConnectionResult.failed("the lookup path is not a directory: " + directory); + } + // Metadata access alone does not prove this user can browse paths in the policy editor. + files.listStatusIterator(new Path(directory)).hasNext(); + return ConnectionResult.succeeded(); + }); } catch (IOException | IllegalArgumentException failed) { return ConnectionResult.failed(message(failed)); @@ -162,17 +238,38 @@ public List lookup(LookupRequest request) if (!PATH.equals(request.resource())) { return List.of(); } + String root = lookupRoot(request.config()); String typed = request.userInput().strip(); - String path = typed.startsWith("/") ? typed : "/" + typed; + String path = typed.isEmpty() ? root + "/" : typed.startsWith("/") ? typed : ("/".equals(root) ? "/" : root + "/") + typed; + boolean children = path.endsWith("/"); + path = normalizePath(path); + if (!"/".equals(root) && !path.equals(root) && !path.startsWith(root + "/")) { + throw new IllegalArgumentException("lookup path must be inside " + root); + } + if (children || path.equals(root)) { + path = "/".equals(path) ? path : path + "/"; + } int slash = path.lastIndexOf('/'); String directory = slash == 0 ? "/" : path.substring(0, slash); String prefix = path.substring(slash + 1); String base = "/".equals(directory) ? "" : directory; + int maximum = scanLimit(request.config()); List entries; try { entries = new HadoopClient(request.config()).run(files -> { try { - return Arrays.asList(files.listStatus(new Path(directory))); + if (!files.getFileStatus(new Path(directory)).isDirectory()) { + return List.of(); + } + RemoteIterator iterator = files.listStatusIterator(new Path(directory)); + List found = new ArrayList<>(); + while (iterator.hasNext()) { + if (found.size() >= maximum) { + throw new IOException("directory exceeds " + maximum + " entries; narrow the lookup directory using " + LOOKUP_ROOT); + } + found.add(iterator.next()); + } + return found; } catch (FileNotFoundException missing) { return List.of(); diff --git a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/FakeWebHdfs.java b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/FakeWebHdfs.java index 77aa59ff..b865406e 100644 --- a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/FakeWebHdfs.java +++ b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/FakeWebHdfs.java @@ -62,7 +62,7 @@ private void handle(HttpExchange exchange) throws IOException + "\"org.apache.hadoop.ipc.StandbyException\",\"message\":\"Operation category READ is not supported in state standby\"}}"); return; } - if (query.contains("user.name=nobody")) { + if (query.contains("user.name=nobody") || query.contains("user.name=stat-only") && query.contains("op=LISTSTATUS")) { respond(exchange, 403, "{\"RemoteException\":{\"exception\":\"AccessControlException\",\"javaClassName\":" + "\"org.apache.hadoop.security.AccessControlException\",\"message\":\"Permission denied: user=nobody\"}}"); return; @@ -86,7 +86,9 @@ private void handle(HttpExchange exchange) throws IOException for (String[] entry : TREE.getOrDefault(normal, List.of())) { statuses.append(statuses.length() == 0 ? "" : ",").append(status(entry[0], entry[1])); } - respond(exchange, 200, "{\"FileStatuses\":{\"FileStatus\":[" + statuses + "]}}"); + String listing = "{\"FileStatuses\":{\"FileStatus\":[" + statuses + "]}}"; + respond(exchange, 200, query.contains("op=LISTSTATUS_BATCH") + ? "{\"DirectoryListing\":{\"partialListing\":" + listing + ",\"remainingEntries\":0}}" : listing); } /** A FileStatus with every field Hadoop's WebHDFS client reads. */ diff --git a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HadoopClientTest.java b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HadoopClientTest.java index d3c764f2..24760e2f 100644 --- a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HadoopClientTest.java +++ b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HadoopClientTest.java @@ -6,12 +6,21 @@ package org.devlive.grantforge.hdfs; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.junit.jupiter.api.Test; +import java.io.IOException; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; +import static org.assertj.core.api.Assertions.assertThatThrownBy; class HadoopClientTest { @@ -29,6 +38,10 @@ void readsAdditionalPropertiesLineByLine() Map.entry("dfs.namenode.rpc-address.ns1.nn1", "a:8020=x")); assertThatIllegalArgumentException().isThrownBy(() -> HadoopClient.properties("a=b\n=c")).withMessageContaining("line 2"); assertThatIllegalArgumentException().isThrownBy(() -> HadoopClient.properties("plain")).withMessageContaining("line 1"); + assertThatIllegalArgumentException().isThrownBy(() -> HadoopClient.properties("a=b\na=c")) + .withMessageContaining("line 2").withMessageContaining("repeats property a"); + assertThatIllegalArgumentException().isThrownBy(() -> HadoopClient.properties("a b=c")) + .withMessageContaining("line 1").withMessageContaining("whitespace"); } @Test @@ -49,4 +62,85 @@ void turnsTheServiceIntoAHadoopConfiguration() assertThat(hadoop.get("ipc.client.connect.timeout")).isEqualTo("5000"); assertThat(hadoop.get("fs.hdfs.impl.disable.cache")).isEqualTo("true"); } + + @Test + void defaultsToSimpleAuthenticationAndKeepsAdditionalSettingsAuthoritative() + { + Configuration defaults = new HadoopClient(HdfsProviderTest.config("hdfs://ns1")).configuration(); + assertThat(defaults.get(HdfsProvider.AUTHENTICATION)).isEqualTo("simple"); + assertThat(defaults.get(HdfsProvider.AUTHORIZATION)).isEqualTo("false"); + + Configuration overridden = new HadoopClient(HdfsProviderTest.config("hdfs://ns1", HdfsProvider.EXTRA, + "hadoop.security.authentication=kerberos\nhadoop.security.authorization=true\nfs.defaultFS=hdfs://ns2")) + .configuration(); + assertThat(overridden.get(HdfsProvider.AUTHENTICATION)).isEqualTo("kerberos"); + assertThat(overridden.get(HdfsProvider.AUTHORIZATION)).isEqualTo("true"); + assertThat(overridden.get("fs.defaultFS")).isEqualTo("hdfs://ns2"); + } + + @Test + void rejectsUnsupportedEffectiveAuthentication() + { + HadoopClient client = new HadoopClient(HdfsProviderTest.config("file:///", HdfsProvider.EXTRA, + "hadoop.security.authentication=unknown")); + + assertThatThrownBy(() -> client.run(files -> true)).isInstanceOf(IOException.class) + .hasMessageContaining("unsupported Hadoop authentication type: unknown"); + } + + @Test + void cancelsAnInterruptedCallAndPreservesTheInterrupt() + { + HadoopClient client = new HadoopClient(HdfsProviderTest.config("file:///")); + Thread.currentThread().interrupt(); + try { + assertThatThrownBy(() -> client.run(files -> true)).isInstanceOf(IOException.class) + .hasCauseInstanceOf(InterruptedException.class); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } + finally { + Thread.interrupted(); + } + } + + @Test + void isolatesTheSecurityConfigurationUntilTheFileSystemActionFinishes() throws Exception + { + ExecutorService workers = Executors.newFixedThreadPool(2); + CountDownLatch insideFirst = new CountDownLatch(1); + CountDownLatch finishFirst = new CountDownLatch(1); + CountDownLatch startedSecond = new CountDownLatch(1); + try { + Future first = workers.submit(() -> new HadoopClient(HdfsProviderTest.config("file:///", "username", "first")) + .run(files -> { + insideFirst.countDown(); + try { + if (!finishFirst.await(5, TimeUnit.SECONDS)) { + throw new IOException("the first call was not released"); + } + } + catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IOException("interrupted in the first call", interrupted); + } + return UserGroupInformation.getCurrentUser().getUserName(); + })); + assertThat(insideFirst.await(5, TimeUnit.SECONDS)).isTrue(); + Future second = workers.submit(() -> { + startedSecond.countDown(); + return new HadoopClient(HdfsProviderTest.config("file:///", "username", "second")) + .run(files -> UserGroupInformation.getCurrentUser().getUserName()); + }); + assertThat(startedSecond.await(5, TimeUnit.SECONDS)).isTrue(); + assertThatThrownBy(() -> second.get(200, TimeUnit.MILLISECONDS)).isInstanceOf(TimeoutException.class); + finishFirst.countDown(); + assertThat(first.get(5, TimeUnit.SECONDS)).isEqualTo("first"); + assertThat(second.get(5, TimeUnit.SECONDS)).isEqualTo("second"); + } + finally { + finishFirst.countDown(); + workers.shutdownNow(); + assertThat(workers.awaitTermination(5, TimeUnit.SECONDS)).isTrue(); + } + } } diff --git a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HdfsProviderTest.java b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HdfsProviderTest.java index 7bc3f748..8d5f8c7d 100644 --- a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HdfsProviderTest.java +++ b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/HdfsProviderTest.java @@ -66,7 +66,7 @@ void declaresPathsWithReadWriteAndExecuteAndRangersSettings() assertThat(definition.configFields()).extracting(field -> field.name()).containsExactly("username", "password", "keytab", "fs.default.name", "hadoop.security.authorization", "hadoop.security.authentication", "hadoop.security.auth_to_local", "dfs.datanode.kerberos.principal", "dfs.namenode.kerberos.principal", "dfs.secondary.namenode.kerberos.principal", - "hadoop.rpc.protection", "hadoop.config"); + "hadoop.rpc.protection", "hadoop.config", "lookup.path", "lookup.max.entries"); } @Test @@ -120,6 +120,88 @@ void speaksWebHdfsAsTheUser() throws IOException } } + @Test + void validatesTheEffectiveClusterAddressAndAuthentication() + { + for (String address : List.of("hdfs:///", "hdfs://user:secret@nn:8020", "hdfs://nn:0", "hdfs://nn:70000", + "webhdfs://nn:9870/data", "hdfs://nn?user=alice", "hdfs://nn#fragment", "HDFS://nn:8020")) { + assertThat(provider.validateConfig(config(address))).as(address).extracting(ConfigProblem::field) + .contains("fs.default.name"); + } + assertThat(provider.validateConfig(config("viewfs:///"))).isEmpty(); + assertThat(provider.validateConfig(config("swebhdfs://nn:9871/"))).isEmpty(); + assertThat(provider.validateConfig(config("hdfs://[::1]:8020"))).isEmpty(); + assertThat(provider.validateConfig(config("hdfs://nn:8020", "hadoop.config", "fs.defaultFS=file:///"))) + .extracting(ConfigProblem::field).containsExactly("hadoop.config"); + assertThat(provider.validateConfig(config("hdfs://nn:8020", "hadoop.config", "fs.default.name=file:///"))) + .extracting(ConfigProblem::field).containsExactly("hadoop.config"); + assertThat(provider.validateConfig(config("hdfs://nn:8020", "hadoop.config", "fs.defaultFS=hdfs://nn\nfs.default.name=file:///"))) + .extracting(ConfigProblem::field).containsExactly("hadoop.config"); + assertThat(provider.validateConfig(config("hdfs://nn:8020", "hadoop.config", "fs.default.name=file:///\nfs.defaultFS=hdfs://nn"))) + .extracting(ConfigProblem::field).containsExactly("hadoop.config"); + assertThat(provider.validateConfig(config("hdfs://nn:8020", "hadoop.config", "hadoop.security.authentication=kerberos"))) + .extracting(ConfigProblem::field).containsExactly("password"); + assertThat(provider.validateConfig(config("hdfs://nn:8020", "hadoop.config", "hadoop.security.authentication=invalid"))) + .extracting(ConfigProblem::field).containsExactly("hadoop.config"); + assertThat(provider.validateConfig(config("hdfs://nn:8020", "hadoop.security.authentication", "kerberos", "hadoop.config", + "hadoop.security.authentication=simple"))).isEmpty(); + } + + @Test + void browsesOnlyTheConfiguredDirectoryAndNormalizesPaths() throws IOException + { + try (FakeWebHdfs namenode = new FakeWebHdfs(false)) { + ServiceConfig config = config("webhdfs://127.0.0.1:" + namenode.uri().getPort(), "lookup.path", "/user/"); + assertThat(provider.testConnection(config)).isEqualTo(ConnectionResult.succeeded()); + assertThat(lookup(config, "")).containsExactly("/user/alice", "/user/bob", "/user/notes.txt"); + assertThat(lookup(config, "a")).containsExactly("/user/alice"); + assertThat(lookup(config, "/user")).containsExactly("/user/alice", "/user/bob", "/user/notes.txt"); + assertThat(lookup(config, "/user//./a")).containsExactly("/user/alice"); + int requests = namenode.requests.size(); + for (String path : List.of("/tmp/", "/users/", "../tmp/", "webhdfs://other/user/", "/user/../tmp/", "/user/\0")) { + assertThatThrownBy(() -> lookup(config, path)).as(path).isInstanceOf(IllegalArgumentException.class); + } + assertThat(namenode.requests).hasSize(requests); + assertThat(lookup(config, "alice/")).isEmpty(); + assertThat(lookup(config, "notes.txt/x")).isEmpty(); + } + } + + @Test + void boundsDirectoryScansAndChecksThatTheLookupPathIsReadable() throws IOException + { + try (FakeWebHdfs namenode = new FakeWebHdfs(false)) { + String address = "webhdfs://127.0.0.1:" + namenode.uri().getPort(); + ServiceConfig config = config(address, "lookup.path", "/user", "lookup.max.entries", "2"); + assertThatThrownBy(() -> lookup(config, "a")).isInstanceOf(UncheckedIOException.class) + .hasMessageContaining("directory exceeds 2 entries"); + assertThat(lookup(config(address, "lookup.path", "/user", "lookup.max.entries", "3"), "a")) + .containsExactly("/user/alice"); + assertThat(provider.testConnection(config(address, "lookup.path", "/user/notes.txt")).message()) + .contains("not a directory"); + assertThat(provider.testConnection(config(address, "lookup.path", "/missing")).status()) + .isEqualTo(ConnectionResult.Status.FAILED); + // A user may be allowed to stat a directory while being unable to enumerate its entries. + assertThat(provider.testConnection(config(address, "username", "stat-only")).message()).contains("Permission denied"); + } + } + + @Test + void validatesLookupSettingsBeforeConnecting() + { + for (String root : List.of("relative", "/user/../", "hdfs://nn/data", "/data/\0")) { + ServiceConfig config = config("hdfs://nn:8020", "lookup.path", root); + assertThat(provider.validateConfig(config)).extracting(ConfigProblem::field).containsExactly("lookup.path"); + assertThat(provider.testConnection(config).status()).isEqualTo(ConnectionResult.Status.FAILED); + } + for (String limit : List.of("0", "-1", "100001", "wrong")) { + assertThat(provider.validateConfig(config("hdfs://nn:8020", "lookup.max.entries", limit))) + .extracting(ConfigProblem::field).containsExactly("lookup.max.entries"); + } + assertThat(provider.validateConfig(config("hdfs://nn:8020", "lookup.path", " ", "lookup.max.entries", "1"))).isEmpty(); + assertThat(provider.validateConfig(config("hdfs://nn:8020", "lookup.max.entries", "100000"))).isEmpty(); + } + @Test void reportsClustersThatCannotBeReached() { diff --git a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/KerberosLoginTest.java b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/KerberosLoginTest.java index 96f40a7f..bf0ebe7b 100644 --- a/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/KerberosLoginTest.java +++ b/plugins/grantforge-plugin-hdfs/src/test/java/org/devlive/grantforge/hdfs/KerberosLoginTest.java @@ -98,6 +98,16 @@ void signsInWithAPassword() assertThat(provider.testConnection(kerberos(PASSWORD_PRINCIPAL, "password", SECRET))).isEqualTo(ConnectionResult.succeeded()); } + @Test + void signsInWithTheEffectiveAuthenticationConfiguration() + { + assertThat(provider.testConnection(kerberos(PASSWORD_PRINCIPAL, "password", SECRET, + "hadoop.security.authentication", "simple", "hadoop.config", "hadoop.security.authentication=kerberos"))) + .isEqualTo(ConnectionResult.succeeded()); + assertThat(provider.testConnection(kerberos(PASSWORD_PRINCIPAL, + "hadoop.config", "hadoop.security.authentication=simple"))).isEqualTo(ConnectionResult.succeeded()); + } + @Test void reportsRefusedCredentials() { From 0c1090945055d7a7701ff33ec14e2ac8187b9e72 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 07:12:27 -0400 Subject: [PATCH 15/22] feat(agent): let agents credit GrantForge for a strict default deny An agent that denies access without a matching policy still enforces GrantForge's rules, so its access events say so. The undetermined decision before a snapshot is now public for agents to use. --- .../org/devlive/grantforge/agent/AccessEvent.java | 12 ++++++++++++ .../org/devlive/grantforge/agent/AgentDecision.java | 7 ++++++- .../devlive/grantforge/agent/AccessEventTest.java | 12 ++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AccessEvent.java b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AccessEvent.java index e959941f..71aa5db3 100644 --- a/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AccessEvent.java +++ b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AccessEvent.java @@ -140,6 +140,18 @@ public Builder decidedBy(AgentDecision decision) return this; } + /** + * Credits GrantForge for enforcing access without a matching policy, such as an agent's strict default deny. + * Call after {@link #decidedBy} when retaining the snapshot version of an undetermined decision. + * + * @return this builder + */ + public Builder enforcedByGrantForge() + { + this.byGrantForge = true; + return this; + } + /** * Sets when it happened; now by default. * diff --git a/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AgentDecision.java b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AgentDecision.java index 0adc7c1d..7915989a 100644 --- a/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AgentDecision.java +++ b/core/grantforge-agent-core/src/main/java/org/devlive/grantforge/agent/AgentDecision.java @@ -55,7 +55,12 @@ static AgentDecision notDetermined(long policyVersion) return new AgentDecision(Outcome.NOT_DETERMINED, null, policyVersion); } - static AgentDecision withoutSnapshot() + /** + * Returns an undetermined decision before an agent has an applied snapshot. + * + * @return the decision, without a policy id or version + */ + public static AgentDecision withoutSnapshot() { return WITHOUT_SNAPSHOT; } diff --git a/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AccessEventTest.java b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AccessEventTest.java index dfc687e2..257bf7ef 100644 --- a/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AccessEventTest.java +++ b/core/grantforge-agent-core/src/test/java/org/devlive/grantforge/agent/AccessEventTest.java @@ -44,4 +44,16 @@ void creditsTheSystemWhenGrantForgeDidNotDecide() assertThat(AccessEvent.builder("a", "r", "t", true).build().eventId()) .isNotEqualTo(AccessEvent.builder("a", "r", "t", true).build().eventId()); } + + @Test + void creditsGrantForgeWhenItsAgentEnforcesStrictDefaultDeny() + { + Map fields = AccessEvent.builder("bob", "/data", "read", false).decidedBy(AgentDecision.notDetermined(4)) + .enforcedByGrantForge().build().fields(); + + assertThat(fields).containsEntry("outcome", "DENIED").containsEntry("enforcer", "GRANTFORGE") + .containsEntry("policyId", null).containsEntry("policyVersion", 4L); + assertThat(AccessEvent.builder("bob", "/data", "read", false).enforcedByGrantForge().build().fields()) + .containsEntry("enforcer", "GRANTFORGE").containsEntry("policyId", null).containsEntry("policyVersion", null); + } } From 1445466e350fe3d86119d874c75683fbacff6ade Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 07:12:27 -0400 Subject: [PATCH 16/22] feat(hdfs): add the NameNode agent for Hadoop 3.5.0 grantforge-agent-hdfs authorizes HDFS access inside the NameNode with the signed local policies of the agent core and ships access events; snapshot paths are also checked against the paths they copy. The release carries it in agents/hdfs, and the documentation has a page on deploying it. --- configure/assembly/server.xml | 5 + core/grantforge-server/pom.xml | 18 + docs/content/changelog/rebuild.md | 2 +- docs/content/guide/data-services.md | 11 +- docs/content/guide/hdfs-agent.md | 86 +++ docs/lib/navigation.ts | 1 + plugins/grantforge-agent-hdfs/pom.xml | 144 +++++ .../hdfs/agent/HdfsAccessControlEnforcer.java | 387 +++++++++++++ .../hdfs/agent/HdfsAgentSettings.java | 92 +++ .../hdfs/agent/HdfsAuthorizationProvider.java | 163 ++++++ .../grantforge/hdfs/agent/package-info.java | 10 + .../agent/HdfsAccessControlEnforcerTest.java | 530 ++++++++++++++++++ .../hdfs/agent/HdfsAgentSettingsTest.java | 121 ++++ .../agent/HdfsAuthorizationProviderTest.java | 159 ++++++ pom.xml | 1 + 15 files changed, 1727 insertions(+), 3 deletions(-) create mode 100644 docs/content/guide/hdfs-agent.md create mode 100644 plugins/grantforge-agent-hdfs/pom.xml create mode 100644 plugins/grantforge-agent-hdfs/src/main/java/org/devlive/grantforge/hdfs/agent/HdfsAccessControlEnforcer.java create mode 100644 plugins/grantforge-agent-hdfs/src/main/java/org/devlive/grantforge/hdfs/agent/HdfsAgentSettings.java create mode 100644 plugins/grantforge-agent-hdfs/src/main/java/org/devlive/grantforge/hdfs/agent/HdfsAuthorizationProvider.java create mode 100644 plugins/grantforge-agent-hdfs/src/main/java/org/devlive/grantforge/hdfs/agent/package-info.java create mode 100644 plugins/grantforge-agent-hdfs/src/test/java/org/devlive/grantforge/hdfs/agent/HdfsAccessControlEnforcerTest.java create mode 100644 plugins/grantforge-agent-hdfs/src/test/java/org/devlive/grantforge/hdfs/agent/HdfsAgentSettingsTest.java create mode 100644 plugins/grantforge-agent-hdfs/src/test/java/org/devlive/grantforge/hdfs/agent/HdfsAuthorizationProviderTest.java diff --git a/configure/assembly/server.xml b/configure/assembly/server.xml index 88579763..dd6a0da3 100644 --- a/configure/assembly/server.xml +++ b/configure/assembly/server.xml @@ -28,6 +28,11 @@ target/bundled-plugins plugins + + + target/bundled-agents + agents + ${project.parent.basedir}/script/bin bin diff --git a/core/grantforge-server/pom.xml b/core/grantforge-server/pom.xml index eb565f14..16fab4a2 100644 --- a/core/grantforge-server/pom.xml +++ b/core/grantforge-server/pom.xml @@ -214,6 +214,24 @@ + + bundled-agents + prepare-package + + copy + + + + + org.devlive.grantforge + grantforge-agent-hdfs + ${project.version} + jar + ${project.build.directory}/bundled-agents/hdfs + + + + diff --git a/docs/content/changelog/rebuild.md b/docs/content/changelog/rebuild.md index 957e6534..8a907b01 100644 --- a/docs/content/changelog/rebuild.md +++ b/docs/content/changelog/rebuild.md @@ -50,7 +50,7 @@ description: 从零重写的 GrantForge：多租户身份、资源与角色授 - 插件化的服务类型：插件定义资源层级、访问类型、脱敏与行过滤；每个插件独立加载。 - 通用策略编辑器、Ed25519 签名的策略快照、代理心跳与访问审计。 -- 示例插件；HDFS 与 Hive 插件正在开发中。 +- 示例插件、HDFS 服务类型和 Hadoop 3.5.0 NameNode 代理；Hive 插件与其他 Hadoop 版本的代理正在开发中。 ## 质量 diff --git a/docs/content/guide/data-services.md b/docs/content/guide/data-services.md index 13fa1e7e..8254f6e9 100644 --- a/docs/content/guide/data-services.md +++ b/docs/content/guide/data-services.md @@ -12,7 +12,7 @@ description: 用插件管理 HDFS、Hive 等外部系统的权限：数据服务 “数据权限”分组管理 GrantForge 以外的数据系统的权限，架构类似 Apache Ranger：插件定义服务类型，管理员在控制台写策略，部署在目标系统里的代理下载策略并在本地判定访问。 > [!NOTE] -> 当前版本提供插件框架、通用策略编辑器、策略分发与访问审计，以及一个示例插件（`example`）。HDFS 与 Hive 的官方插件和代理正在开发中。 +> 当前版本提供插件框架、通用策略编辑器、策略分发与访问审计、HDFS 服务类型和 Hadoop 3.5.0 NameNode 代理，以及示例插件（`example`）。Hive 插件和其他 Hadoop 版本的代理仍在开发中。 ```mermaid flowchart LR @@ -34,7 +34,8 @@ flowchart LR - 资源只有一级 `path`，按路径匹配：`/data/sales` 匹配它本身，勾选“递归”后也匹配其下的全部文件与目录；支持排除。 - 访问类型 `read`、`write`、`execute`，与 HDFS 的权限位对应。 -- 插件用 Hadoop 自己的客户端连接集群，测试连接读取根目录，写策略时输入路径会列出对应目录下的子目录与文件。 +- 插件用 Hadoop 自己的客户端连接集群，测试连接检查查询目录存在且可以列出内容，写策略时输入路径会列出对应目录下的子目录与文件，目录排在文件前面。 +- 服务端插件负责管理与查询；要让策略约束 HDFS 访问，还需部署 [NameNode 代理](/guide/hdfs-agent/)。 | 配置 | 说明 | | --- | --- | @@ -46,6 +47,12 @@ flowchart LR | `dfs.namenode.kerberos.principal` 等 | NameNode、DataNode、Secondary NameNode 的 principal，如 `nn/_HOST@EXAMPLE.COM` | | `hadoop.rpc.protection` | `authentication`、`integrity` 或 `privacy`，与集群一致 | | 附加 Hadoop 配置 | 每行一个 `key=value`，用于高可用等其他配置，例如 `dfs.nameservices=nameservice1`、`dfs.ha.namenodes.nameservice1=nn1,nn2`、`dfs.namenode.rpc-address.nameservice1.nn1=nn1:8020`、`dfs.client.failover.proxy.provider.nameservice1=org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider` | +| `lookup.path` | 查询目录，默认 `/`；例如设为 `/data` 后，空输入列出 `/data` 的内容，相对输入从这里开始补全。适用于查询用户没有根目录列出权限的集群 | +| `lookup.max.entries` | 一次目录查询最多扫描的条目数，默认 `10000`，范围 `1..100000`；超过上限返回错误，避免静默遗漏候选 | + +附加 Hadoop 配置覆盖同名连接设置，配置校验与登录均使用覆盖后的值。`fs.defaultFS` 和 `fs.default.name` 是别名，附加配置中只能设置其中一个；重复键、非集群地址和无凭据的 Kerberos 配置会在保存时被拒绝。集群地址只填写集群 URI，需要查询的子目录放在 `lookup.path`。 + +`lookup.path` 限制路径候选的浏览范围，不替代 HDFS 自身的访问控制；符号链接和 ViewFS 挂载仍遵循集群配置。输入中可省略开头的 `/`，允许重复 `/` 与 `.`，拒绝 `..` 和范围之外的绝对路径。不存在的目录返回空候选，权限不足和连接失败会显示错误。 使用 Kerberos 时，GrantForge 服务器需要能找到 KDC：配置 `/etc/krb5.conf`，或用 `-Djava.security.krb5.conf=` 指定。 diff --git a/docs/content/guide/hdfs-agent.md b/docs/content/guide/hdfs-agent.md new file mode 100644 index 00000000..56a1b17e --- /dev/null +++ b/docs/content/guide/hdfs-agent.md @@ -0,0 +1,86 @@ +--- +title: HDFS NameNode 代理 +description: 在 Hadoop 3.5.0 NameNode 内执行 GrantForge 路径策略，并上报访问审计。 +--- + + +服务端的 `grantforge-plugin-hdfs` 定义资源和连接配置；`grantforge-agent-hdfs` 安装在 NameNode 内，通过 Hadoop 的 `INodeAttributeProvider` 与 `AccessControlEnforcer` 检查访问，复用 GrantForge 代理核心下载签名策略、保存本地快照和批量上报审计。当前代理针对 **Hadoop 3.5.0、Java 17 及以上**构建，其他 Hadoop 版本需要对应版本的适配和验证。 + +## 权限关系 + +代理先执行 HDFS 原生权限检查，再执行 GrantForge 策略：用户需要同时满足原生权限与策略要求。GrantForge 的允许策略不会绕过 POSIX 权限、ACL、所有者检查或 sticky bit；拒绝策略始终拒绝。原生权限设置仍通过 Hadoop 的管理工具维护。 + +默认 `grantforge.hdfs.native.fallback=false`：没有本地策略快照、没有匹配策略或代理尚未启动时拒绝数据访问。设为 `true` 后，未被策略决定的访问使用原生权限；显式拒绝策略仍然有效。服务端暂时不可达时继续使用最后一份通过签名验证的本地快照。 + +代理检查普通用户访问目标所需的 `read`、`write`、`execute`，也检查父目录、祖先目录与需要递归校验的子目录。创建、删除、重命名等操作涉及多个路径，允许策略必须覆盖它们。严格模式下，只有目标文件的 `read` 策略还不够，需要给用户配置祖先目录的 `execute` 策略，例如允许 `/` 上的 `execute` 并勾选递归，再为实际数据目录配置读写权限。 + +快照路径同时检查实际请求路径和去掉 `.snapshot/<快照名>` 后的原路径，例如 `/data/.snapshot/s1/secret` 同时检查 `/data/secret`。原路径上的拒绝策略因此也约束快照；可以再为显式快照路径设置更严格的限制。元数据查询沿用 HDFS 的目录遍历权限语义。 + +一次递归授权最多检查 `100000` 个 inode，超过上限会拒绝操作，避免在 NameNode 内无限分配内存。超长路径使用完整路径判定策略；审计资源展示限制为 `1000` 字符，并在请求详情中记录原长度和 SHA-256 摘要。 + +HDFS 超级用户仍由 Hadoop 管理。Hadoop 的超级用户回调没有完整 inode 与子树上下文：带路径的调用保守地要求该路径的三种权限，无路径的集群管理调用保留原生检查；无法用子目录策略限制超级用户的所有递归操作。数据使用者应使用普通 Hadoop 用户。 + +## 部署 + +1. 在 GrantForge 的数据服务中添加 `hdfs` 服务，保存配置并测试连接；为实际 Hadoop 短用户名、用户组或角色配置路径策略。 +2. 在“数据权限 → 代理”中为这个服务签发令牌。将令牌原文写到每个 NameNode 的本地文件，例如 `/etc/hadoop/grantforge/token`，由 NameNode 运行用户读取。 +3. 将发行包 `agents/hdfs/grantforge-agent-hdfs-2026.0.0.jar` 放入 NameNode 的类路径，例如 `$HADOOP_HOME/share/hadoop/hdfs/lib/`。代理 jar 已包含自己的策略引擎、Jackson 和签名库，Hadoop 类由 NameNode 提供。 +4. 在每个 NameNode 的 `hdfs-site.xml` 中配置以下属性；HA 的两个 NameNode 使用不同的 `instance` 和各自本地的缓存目录。 + +```xml + + dfs.namenode.inode.attributes.provider.class + org.devlive.grantforge.hdfs.agent.HdfsAuthorizationProvider + + + grantforge.hdfs.server.url + https://grantforge.example.com/ + + + grantforge.hdfs.token.file + /etc/hadoop/grantforge/token + + + grantforge.hdfs.instance + namenode-1 + + + grantforge.hdfs.cache.dir + /var/lib/hadoop/grantforge + + + grantforge.hdfs.native.fallback + false + +``` + +5. 确认 `dfs.permissions.enabled=true`，且 `dfs.namenode.inode.attributes.provider.bypass.users` 为空；代理启动时会拒绝可绕过授权回调的配置。重启 NameNode，然后检查 GrantForge 代理页面中的心跳与策略版本。代理读取 NameNode 现有配置；不修改 inode 的原生属性。 + +首次部署可先使用 `native.fallback=true`，确认策略快照已同步并补齐祖先目录权限，再切换严格模式。令牌绑定的服务类型必须是 `hdfs`；配置错误或绑定到其他服务类型会拒绝访问。 + +## 可选设置 + +| 属性 | 默认值 | 用途 | +| --- | --- | --- | +| `grantforge.hdfs.connect.timeout.ms` | `5000` | 连接 GrantForge 的超时 | +| `grantforge.hdfs.read.timeout.ms` | `8000` | 读取响应的超时 | +| `grantforge.hdfs.refresh.interval.ms` | `30000` | 服务器不可达时的策略刷新间隔，至少 `1000`；正常心跳使用服务端建议间隔 | +| `grantforge.hdfs.signing.key.file` | 未设置 | 可选签名公钥文件，内容为控制台提供的 Base64 X.509 公钥；配置后只接受该公钥的签名 | + +未配置签名公钥时，代理首次从服务器获取公钥并随快照保存。代理使用 Hadoop 传入的短用户名和用户组，角色及额外组来自签名快照；Kerberos principal 的短名映射由集群的 `hadoop.security.auth_to_local` 决定。 + +## 从源码构建与验证 + +```sh +./mvnw -pl plugins/grantforge-agent-hdfs -am package +./mvnw -pl plugins/grantforge-plugin-hdfs,plugins/grantforge-agent-hdfs,core/grantforge-plugin-host -am test +``` + +代理产物位于 `plugins/grantforge-agent-hdfs/target/grantforge-agent-hdfs-2026.0.0.jar`。单元测试覆盖 NameNode 授权回调、配置和策略决策；WebHDFS 与 Kerberos 测试启动本机临时服务。上线前还需在目标集群验证读写、创建、重命名、递归删除、HA 切换和断网后的缓存行为。 + +Hadoop 扩展入口与权限语义见 [Apache Hadoop 3.5.0 API](https://hadoop.apache.org/docs/r3.5.0/hadoop-project-dist/hadoop-hdfs/build/source/hadoop-hdfs-project/hadoop-hdfs/target/api/org/apache/hadoop/hdfs/server/namenode/INodeAttributeProvider.html) 和 [HDFS 权限指南](https://hadoop.apache.org/docs/r3.5.0/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html)。 diff --git a/docs/lib/navigation.ts b/docs/lib/navigation.ts index 3688156b..d0bba996 100644 --- a/docs/lib/navigation.ts +++ b/docs/lib/navigation.ts @@ -66,6 +66,7 @@ export const sections: NavSection[] = [ { slug: 'guide/tenants', title: '租户' }, { slug: 'guide/catalog', title: '资源目录与 API 目录' }, { slug: 'guide/data-services', title: '数据服务、策略与代理' }, + { slug: 'guide/hdfs-agent', title: 'HDFS NameNode 代理' }, ] }, ], }, diff --git a/plugins/grantforge-agent-hdfs/pom.xml b/plugins/grantforge-agent-hdfs/pom.xml new file mode 100644 index 00000000..97214351 --- /dev/null +++ b/plugins/grantforge-agent-hdfs/pom.xml @@ -0,0 +1,144 @@ + + + + + 4.0.0 + + org.devlive.grantforge + grantforge + 2026.0.0 + ../../pom.xml + + grantforge-agent-hdfs + GrantForge HDFS NameNode Agent + Hadoop 3.5.0 NameNode authorization overlay, signed local policies and access auditing. Java 17. + + + + + + + org.devlive.grantforge + grantforge-agent-core + + + + org.apache.hadoop + hadoop-client-api + provided + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + provided + + ** + + + + org.jspecify + jspecify + provided + + + org.apache.hadoop + hadoop-client-runtime + test + + + org.springframework.boot + spring-boot-starter-test + test + + + + org.apache.hadoop.thirdparty + hadoop-shaded-guava + 1.5.0 + test + + + commons-cli + commons-cli + 1.9.0 + test + + + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 + 1.5.0 + test + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + test-agent-path + initialize + properties + + + + + org.apache.maven.plugins + maven-surefire-plugin + + @{argLine} -javaagent:${org.mockito:mockito-core:jar} + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.1 + + + package + shade + + false + + + com.fasterxml.jackson + org.devlive.grantforge.hdfs.agent.internal.jackson + + + org.bouncycastle + org.devlive.grantforge.hdfs.agent.internal.bouncycastle + + + + + *:* + + META-INF/*.SF + META-INF/*.RSA + META-INF/*.DSA + module-info.class + META-INF/versions/*/module-info.class + + + + + + + + + + + + + diff --git a/plugins/grantforge-agent-hdfs/src/main/java/org/devlive/grantforge/hdfs/agent/HdfsAccessControlEnforcer.java b/plugins/grantforge-agent-hdfs/src/main/java/org/devlive/grantforge/hdfs/agent/HdfsAccessControlEnforcer.java new file mode 100644 index 00000000..7109a948 --- /dev/null +++ b/plugins/grantforge-agent-hdfs/src/main/java/org/devlive/grantforge/hdfs/agent/HdfsAccessControlEnforcer.java @@ -0,0 +1,387 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.hdfs.agent; + +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.hdfs.server.namenode.INode; +import org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer; +import org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AuthorizationContext; +import org.apache.hadoop.hdfs.server.namenode.INodeAttributes; +import org.apache.hadoop.hdfs.util.ReadOnlyList; +import org.apache.hadoop.ipc.CallerContext; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; +import org.devlive.grantforge.agent.AccessEvent; +import org.devlive.grantforge.agent.AgentDecision; +import org.devlive.grantforge.policy.engine.AccessRequest; +import org.jspecify.annotations.Nullable; + +import java.net.InetAddress; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BooleanSupplier; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** Adds path policy checks only after Hadoop has checked traversal, ACLs, ownership, sticky bits and subtrees. */ +final class HdfsAccessControlEnforcer + implements AccessControlEnforcer +{ + private static final Logger LOG = Logger.getLogger(HdfsAccessControlEnforcer.class.getName()); + private static final int MAX_SUBTREE_ENTRIES = 100_000; + private final @Nullable AccessControlEnforcer nativeEnforcer; + private final Function decide; + private final Consumer record; + private final BooleanSupplier nativeFallback; + + HdfsAccessControlEnforcer(@Nullable AccessControlEnforcer nativeEnforcer, Function decide, + Consumer record, BooleanSupplier nativeFallback) + { + this.nativeEnforcer = nativeEnforcer; + this.decide = decide; + this.record = record; + this.nativeFallback = nativeFallback; + } + + @Override + @SuppressWarnings("deprecation") + public void checkPermission(String fsOwner, String supergroup, UserGroupInformation callerUgi, INodeAttributes[] inodeAttrs, + INode[] inodes, byte[][] pathByNameArr, int snapshotId, @Nullable String path, int ancestorIndex, + boolean doCheckOwner, @Nullable FsAction ancestorAccess, @Nullable FsAction parentAccess, @Nullable FsAction access, + @Nullable FsAction subAccess, boolean ignoreEmptyDir) throws AccessControlException + { + AuthorizationContext context = new AuthorizationContext.Builder().fsOwner(fsOwner).supergroup(supergroup) + .callerUgi(callerUgi).inodeAttrs(inodeAttrs).inodes(inodes).pathByNameArr(pathByNameArr).snapshotId(snapshotId) + .path(path).ancestorIndex(ancestorIndex).doCheckOwner(doCheckOwner).ancestorAccess(ancestorAccess) + .parentAccess(parentAccess).access(access).subAccess(subAccess).ignoreEmptyDir(ignoreEmptyDir).build(); + AccessControlEnforcer nativeChecks = nativeChecks(); + try { + nativeChecks.checkPermission(fsOwner, supergroup, callerUgi, inodeAttrs, inodes, pathByNameArr, snapshotId, path, + ancestorIndex, doCheckOwner, ancestorAccess, parentAccess, access, subAccess, ignoreEmptyDir); + } + catch (AccessControlException denied) { + nativeEvent(context, false, denied.getMessage()); + throw denied; + } + enforce(context); + } + + @Override + public void checkPermissionWithContext(AuthorizationContext context) throws AccessControlException + { + try { + nativeChecks().checkPermissionWithContext(context); + } + catch (AccessControlException denied) { + nativeEvent(context, false, denied.getMessage()); + throw denied; + } + enforce(context); + } + + @Override + public void checkSuperUserPermissionWithContext(AuthorizationContext context) throws AccessControlException + { + try { + nativeChecks().checkSuperUserPermissionWithContext(context); + } + catch (AccessControlException denied) { + nativeEvent(context, false, denied.getMessage()); + throw denied; + } + String path = context.getPath(); + if (path == null) { + // Pathless cluster administration has no resource in the HDFS service model. Hadoop's privilege gate applies. + nativeEvent(context, true, null); + return; + } + Map permissions = new LinkedHashMap<>(); + add(permissions, path, FsAction.ALL); + evaluate(context, permissions.keySet().stream().toList()); + } + + @Override + public void denyUserAccess(AuthorizationContext context, String errorMessage) throws AccessControlException + { + nativeEvent(context, false, errorMessage); + // Never depend on the delegate to throw: this callback means Hadoop already rejected the request. + throw new AccessControlException(errorMessage); + } + + private AccessControlEnforcer nativeChecks() throws AccessControlException + { + AccessControlEnforcer checks = nativeEnforcer; + if (checks == null) { + throw new AccessControlException("Hadoop did not supply its native permission checker"); + } + return checks; + } + + private void enforce(AuthorizationContext context) throws AccessControlException + { + try { + evaluate(context, permissions(context)); + } + catch (AccessControlException denied) { + throw denied; + } + catch (RuntimeException invalid) { + String path = context.getPath(); + event(context, new Permission(path == null ? "/" : path, "execute"), false, null, "invalid authorization context"); + throw failure("GrantForge could not check this HDFS authorization context: " + invalid.getMessage(), invalid); + } + } + + // Each queued inode needs its own path holder; the discovery cap bounds these allocations. + @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") + private static List permissions(AuthorizationContext context) + { + Map permissions = new LinkedHashMap<>(); + INode[] inodes = context.getInodes(); + int last = inodes.length - 1; + if (last < 0) { + throw new IllegalArgumentException("the authorization context has no inode path"); + } + int ancestor = context.getAncestorIndex(); + while (ancestor >= 0 && inodes[ancestor] == null) { + ancestor--; + } + for (int index = 0; index <= ancestor; index++) { + add(permissions, path(context, index), FsAction.EXECUTE); + } + if (last > 0 && ancestor >= 0) { + add(permissions, path(context, ancestor), context.getAncestorAccess()); + } + if (last > 0 && inodes[last - 1] != null) { + add(permissions, path(context, last - 1), context.getParentAccess()); + } + String target = path(context, last); + add(permissions, target, context.getAccess()); + // Creation, removal and rename also restrict the named target, including targets not yet in the inode tree. + add(permissions, target, context.getAncestorAccess()); + add(permissions, target, context.getParentAccess()); + if (context.isDoCheckOwner()) { + add(permissions, target, FsAction.WRITE); + } + FsAction subAccess = context.getSubAccess(); + INode inode = inodes[last]; + if (subAccess != null && inode != null && inode.isDirectory()) { + Deque pending = new ArrayDeque<>(); + pending.push(new Subtree(inode, target)); + int entries = 1; + while (!pending.isEmpty()) { + Subtree subtree = pending.pop(); + // Apply the overlay to empty directories and child files too, so deleting an ancestor cannot bypass a deny. + add(permissions, subtree.path, subAccess); + if (subtree.inode.isDirectory()) { + ReadOnlyList children = subtree.inode.asDirectory().getChildrenList(context.getSnapshotId()); + if (children.size() > MAX_SUBTREE_ENTRIES - entries) { + throw new IllegalArgumentException("subtree exceeds " + MAX_SUBTREE_ENTRIES + + " entries; authorize smaller subtrees separately"); + } + entries += children.size(); + for (INode child : children) { + String name = new String(child.getLocalNameBytes(), StandardCharsets.UTF_8); + pending.push(new Subtree(child, childPath(subtree.path, name))); + } + } + } + } + if (permissions.isEmpty()) { + // Root/NONE checks have no traversed ancestors. Still require a policy decision rather than allowing vacuously. + add(permissions, target, FsAction.EXECUTE); + } + return List.copyOf(permissions.keySet()); + } + + // Each UTF-8 component must be decoded separately before reconstructing the requested path. + @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") + private static String path(AuthorizationContext context, int inodeIndex) + { + if (inodeIndex == context.getInodes().length - 1 && context.getPath() != null) { + return context.getPath(); + } + byte[][] components = context.getPathByNameArr(); + // Hadoop also sends a single inode with the full path's components (content-summary checks). + int componentIndex = inodeIndex + components.length - context.getInodes().length; + StringBuilder path = new StringBuilder(); + for (int index = 0; index <= componentIndex; index++) { + byte[] component = components[index]; + if (component != null && component.length > 0) { + path.append('/').append(new String(component, StandardCharsets.UTF_8)); + } + } + return path.length() == 0 ? "/" : path.toString(); + } + + private static String childPath(String parent, String name) + { + return "/".equals(parent) ? "/" + name : parent + "/" + name; + } + + private static void add(Map permissions, String path, @Nullable FsAction action) + { + if (action == null || action == FsAction.NONE) { + return; + } + if (action.implies(FsAction.READ)) { + Permission permission = new Permission(path, "read"); + permissions.put(permission, permission); + } + if (action.implies(FsAction.WRITE)) { + Permission permission = new Permission(path, "write"); + permissions.put(permission, permission); + } + if (action.implies(FsAction.EXECUTE)) { + Permission permission = new Permission(path, "execute"); + permissions.put(permission, permission); + } + String live = livePath(path); + if (!path.equals(live)) { + add(permissions, live, action); + } + } + + private static String livePath(String path) + { + if (!path.contains("/.snapshot")) { + return path; + } + String[] elements = path.split("/", -1); + StringBuilder live = new StringBuilder(); + boolean skipSnapshotName = false; + for (String element : elements) { + if (skipSnapshotName) { + skipSnapshotName = false; + continue; + } + if (".snapshot".equals(element)) { + skipSnapshotName = true; + } + else if (!element.isEmpty()) { + live.append('/').append(element); + } + } + return live.length() == 0 ? "/" : live.toString(); + } + + private void evaluate(AuthorizationContext context, List permissions) throws AccessControlException + { + String user = context.getCallerUgi().getShortUserName(); + String[] groups = context.getCallerUgi().getGroupNames(); + Instant time = Instant.now(); + Map attributes = new LinkedHashMap<>(); + String clientIp = clientIp(); + if (clientIp != null) { + attributes.put("clientAddress", clientIp); + } + String operation = context.getOperationName(); + if (operation != null) { + attributes.put("operation", operation); + } + List decisions = new ArrayList<>(); + for (Permission permission : permissions) { + AgentDecision decision; + try { + decision = decide.apply(AccessRequest.builder(user, permission.access).resource("path", permission.path) + .groups(groups).time(time).context(attributes).build()); + } + catch (RuntimeException broken) { + event(context, permission, false, null, "policy evaluation failed"); + throw failure("GrantForge could not evaluate HDFS access", broken); + } + boolean allowed = decision.allowed() + || (decision.outcome() == AgentDecision.Outcome.NOT_DETERMINED && nativeFallback.getAsBoolean()); + if (!allowed) { + event(context, permission, false, decision, "GrantForge denied HDFS access"); + throw new AccessControlException("GrantForge denied " + permission.access + " for user " + user + + " on " + permission.path + " (" + decision.outcome() + ")"); + } + decisions.add(decision); + } + // Emit successful checks only after the entire operation's overlay has passed. + for (int index = 0; index < permissions.size(); index++) { + event(context, permissions.get(index), true, decisions.get(index), null); + } + } + + private void nativeEvent(AuthorizationContext context, boolean allowed, @Nullable String reason) + { + String path = context.getPath(); + if (path == null && context.getInodes() != null && context.getInodes().length > 0 && context.getPathByNameArr() != null) { + path = path(context, context.getInodes().length - 1); + } + event(context, new Permission(path == null ? "/" : path, "native"), allowed, null, reason); + } + + private void event(AuthorizationContext context, Permission permission, boolean allowed, @Nullable AgentDecision decision, + @Nullable String reason) + { + String resource = permission.path; + String details = reason; + if (resource.length() > 1000) { + details = "resourceLength=" + resource.length() + "; resourceSha256=" + digest(resource) + + (reason == null ? "" : "; " + reason); + resource = resource.substring(0, 985) + "...[truncated]"; + } + AccessEvent.Builder event = AccessEvent.builder(context.getCallerUgi().getShortUserName(), resource, + permission.access, allowed) + .clientIp(clientIp()).resourceType("path").action(context.getOperationName()); + if (decision != null) { + event.decidedBy(decision); + } + if (!allowed && !"native".equals(permission.access)) { + event.enforcedByGrantForge(); + } + CallerContext caller = context.getCallerContext(); + event.request(details == null ? (caller == null ? null : caller.getContext()) : details); + try { + record.accept(event.build()); + } + catch (RuntimeException unavailable) { + LOG.log(Level.WARNING, "Could not queue an HDFS access audit event", unavailable); + } + } + + private static @Nullable String clientIp() + { + InetAddress remote = Server.getRemoteIp(); + return remote == null ? null : remote.getHostAddress(); + } + + private static AccessControlException failure(String message, RuntimeException cause) + { + AccessControlException denied = new AccessControlException(message); + denied.initCause(cause); + return denied; + } + + private static String digest(String resource) + { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(resource.getBytes(StandardCharsets.UTF_8))); + } + catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 is unavailable", impossible); + } + } + + private record Permission(String path, String access) {} + + private record Subtree(INode inode, String path) {} +} diff --git a/plugins/grantforge-agent-hdfs/src/main/java/org/devlive/grantforge/hdfs/agent/HdfsAgentSettings.java b/plugins/grantforge-agent-hdfs/src/main/java/org/devlive/grantforge/hdfs/agent/HdfsAgentSettings.java new file mode 100644 index 00000000..a07e1661 --- /dev/null +++ b/plugins/grantforge-agent-hdfs/src/main/java/org/devlive/grantforge/hdfs/agent/HdfsAgentSettings.java @@ -0,0 +1,92 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.hdfs.agent; + +import org.apache.hadoop.conf.Configuration; +import org.devlive.grantforge.agent.AgentSettings; +import org.devlive.grantforge.agent.SigningKey; + +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; + +/** Reads the NameNode's agent settings; secrets and an optional pinned signing key come from local files. */ +final class HdfsAgentSettings +{ + static final String PREFIX = "grantforge.hdfs."; + + private final AgentSettings agent; + private final boolean nativeFallback; + + private HdfsAgentSettings(AgentSettings agent, boolean nativeFallback) + { + this.agent = agent; + this.nativeFallback = nativeFallback; + } + + static HdfsAgentSettings read(Configuration configuration) throws IOException + { + if (!configuration.getBoolean("dfs.permissions.enabled", true)) { + throw new IllegalArgumentException("dfs.permissions.enabled must be true for the NameNode to call the agent"); + } + if (!configuration.getTrimmed("dfs.namenode.inode.attributes.provider.bypass.users", "").isEmpty()) { + throw new IllegalArgumentException("dfs.namenode.inode.attributes.provider.bypass.users must be empty for enforcement"); + } + String token = Files.readString(Path.of(required(configuration, "token.file")), StandardCharsets.UTF_8).strip(); + AgentSettings.Builder builder = AgentSettings.builder() + .server(URI.create(required(configuration, "server.url"))) + .token(token) + .instance(required(configuration, "instance")) + .cacheDirectory(Path.of(required(configuration, "cache.dir"))) + .agentVersion("2026.0.0-hadoop-3.5.0") + .timeouts(duration(configuration, "connect.timeout.ms", 5000), duration(configuration, "read.timeout.ms", 8000)) + .refreshInterval(duration(configuration, "refresh.interval.ms", 30000)); + String keyFile = configuration.getTrimmed(PREFIX + "signing.key.file"); + if (keyFile != null && !keyFile.isEmpty()) { + builder.trustedKey(SigningKey.of(Files.readString(Path.of(keyFile), StandardCharsets.US_ASCII))); + } + String fallback = configuration.getTrimmed(PREFIX + "native.fallback", "false"); + if (!"true".equalsIgnoreCase(fallback) && !"false".equalsIgnoreCase(fallback)) { + throw new IllegalArgumentException(PREFIX + "native.fallback must be true or false"); + } + return new HdfsAgentSettings(builder.build(), Boolean.parseBoolean(fallback)); + } + + private static String required(Configuration configuration, String key) + { + String value = configuration.getTrimmed(PREFIX + key); + if (value == null || value.isEmpty()) { + throw new IllegalArgumentException("missing NameNode setting " + PREFIX + key); + } + return value; + } + + private static Duration duration(Configuration configuration, String key, long fallback) + { + long milliseconds = configuration.getLong(PREFIX + key, fallback); + if (milliseconds < 1 || ("refresh.interval.ms".equals(key) && milliseconds < 1000)) { + throw new IllegalArgumentException(PREFIX + key + " must be at least " + + ("refresh.interval.ms".equals(key) ? 1000 : 1) + " milliseconds"); + } + if (!"refresh.interval.ms".equals(key) && milliseconds > Integer.MAX_VALUE) { + throw new IllegalArgumentException(PREFIX + key + " must not exceed " + Integer.MAX_VALUE + " milliseconds"); + } + return Duration.ofMillis(milliseconds); + } + + AgentSettings agent() + { + return agent; + } + + boolean nativeFallback() + { + return nativeFallback; + } +} diff --git a/plugins/grantforge-agent-hdfs/src/main/java/org/devlive/grantforge/hdfs/agent/HdfsAuthorizationProvider.java b/plugins/grantforge-agent-hdfs/src/main/java/org/devlive/grantforge/hdfs/agent/HdfsAuthorizationProvider.java new file mode 100644 index 00000000..ef3af2eb --- /dev/null +++ b/plugins/grantforge-agent-hdfs/src/main/java/org/devlive/grantforge/hdfs/agent/HdfsAuthorizationProvider.java @@ -0,0 +1,163 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.hdfs.agent; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider; +import org.apache.hadoop.hdfs.server.namenode.INodeAttributes; +import org.devlive.grantforge.agent.AgentDecision; +import org.devlive.grantforge.agent.AgentSettings; +import org.devlive.grantforge.agent.GrantForgeAgent; +import org.devlive.grantforge.agent.Snapshot; +import org.devlive.grantforge.policy.engine.AccessRequest; +import org.jspecify.annotations.Nullable; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; + +/** + * Hadoop 3.5.0 NameNode attribute provider. Install its jar in the NameNode classpath and select this class with + * {@code dfs.namenode.inode.attributes.provider.class}. Native inode attributes and HDFS checks remain authoritative; + * GrantForge policies add restrictions. Settings are read from {@code hdfs-site.xml} under {@code grantforge.hdfs.}. + */ +@SuppressWarnings("PMD.AvoidUsingVolatile") +public final class HdfsAuthorizationProvider + extends INodeAttributeProvider + implements Configurable +{ + private final ReentrantLock lifecycle = new ReentrantLock(); + private final Function agentFactory; + private @Nullable Configuration configuration; + private volatile @Nullable GrantForgeAgent agent; + private volatile boolean nativeFallback; + + /** Creates the provider; Hadoop injects the NameNode configuration before starting it. */ + public HdfsAuthorizationProvider() + { + this(settings -> GrantForgeAgent.start(settings, Map.of())); + } + + HdfsAuthorizationProvider(Function agentFactory) + { + this.agentFactory = agentFactory; + } + + @Override + public void setConf(Configuration value) + { + lifecycle.lock(); + try { + if (agent != null) { + throw new IllegalStateException("stop the HDFS agent before changing its configuration"); + } + configuration = new Configuration(value); + } + finally { + lifecycle.unlock(); + } + } + + @Override + public @Nullable Configuration getConf() + { + lifecycle.lock(); + try { + Configuration settings = configuration; + return settings == null ? null : new Configuration(settings); + } + finally { + lifecycle.unlock(); + } + } + + @Override + public void start() + { + lifecycle.lock(); + try { + if (agent != null) { + return; + } + Configuration settings = configuration; + if (settings == null) { + throw new IllegalStateException("the NameNode did not supply the HDFS agent configuration"); + } + try { + HdfsAgentSettings parsed = HdfsAgentSettings.read(settings); + nativeFallback = parsed.nativeFallback(); + agent = agentFactory.apply(parsed.agent()); + } + catch (IOException | IllegalArgumentException invalid) { + throw new IllegalStateException("cannot configure the GrantForge HDFS agent", invalid); + } + } + finally { + lifecycle.unlock(); + } + } + + @Override + // Only this lifecycle method owns closing the agent. Clearing its published reference makes subsequent checks fail closed. + @SuppressWarnings({"PMD.CloseResource", "PMD.NullAssignment"}) + public void stop() + { + lifecycle.lock(); + try { + GrantForgeAgent running = agent; + agent = null; + if (running != null) { + running.close(); + } + } + finally { + lifecycle.unlock(); + } + } + + @Override + public INodeAttributes getAttributes(String[] pathElements, INodeAttributes inode) + { + return inode; + } + + @Override + public AccessControlEnforcer getExternalAccessControlEnforcer(@Nullable AccessControlEnforcer defaultEnforcer) + { + // Hadoop probes the returned class with a null defaultEnforcer during startup, before any permission check. + return new HdfsAccessControlEnforcer(defaultEnforcer, this::decide, this::record, () -> nativeFallback); + } + + // Borrow the provider's shared running agent; stop() alone owns its lifetime and closes it. + @SuppressWarnings("PMD.CloseResource") + private AgentDecision decide(AccessRequest request) + { + GrantForgeAgent running = agent; + if (running == null) { + throw new IllegalStateException("the GrantForge HDFS agent is not running"); + } + Snapshot snapshot = running.snapshot(); + if (snapshot == null) { + return AgentDecision.withoutSnapshot(); + } + if (!"hdfs".equals(snapshot.serviceType())) { + throw new IllegalStateException("the NameNode agent token is bound to a service whose type is not hdfs"); + } + return snapshot.decide(request); + } + + // Audit callbacks borrow the shared agent; they must not close the resource owned by stop(). + @SuppressWarnings("PMD.CloseResource") + private void record(org.devlive.grantforge.agent.AccessEvent event) + { + GrantForgeAgent running = agent; + if (running != null) { + running.record(event); + } + } +} diff --git a/plugins/grantforge-agent-hdfs/src/main/java/org/devlive/grantforge/hdfs/agent/package-info.java b/plugins/grantforge-agent-hdfs/src/main/java/org/devlive/grantforge/hdfs/agent/package-info.java new file mode 100644 index 00000000..17e2c2c3 --- /dev/null +++ b/plugins/grantforge-agent-hdfs/src/main/java/org/devlive/grantforge/hdfs/agent/package-info.java @@ -0,0 +1,10 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +/** Hadoop 3.5.0 NameNode enforcement that retains native HDFS permissions and adds GrantForge policy checks. */ +@NullMarked +package org.devlive.grantforge.hdfs.agent; + +import org.jspecify.annotations.NullMarked; diff --git a/plugins/grantforge-agent-hdfs/src/test/java/org/devlive/grantforge/hdfs/agent/HdfsAccessControlEnforcerTest.java b/plugins/grantforge-agent-hdfs/src/test/java/org/devlive/grantforge/hdfs/agent/HdfsAccessControlEnforcerTest.java new file mode 100644 index 00000000..d9e4d0fd --- /dev/null +++ b/plugins/grantforge-agent-hdfs/src/test/java/org/devlive/grantforge/hdfs/agent/HdfsAccessControlEnforcerTest.java @@ -0,0 +1,530 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.hdfs.agent; + +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.AclEntryScope; +import org.apache.hadoop.fs.permission.AclEntryType; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.server.namenode.AclEntryStatusFormat; +import org.apache.hadoop.hdfs.server.namenode.AclFeature; +import org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker; +import org.apache.hadoop.hdfs.server.namenode.INode; +import org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer; +import org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AuthorizationContext; +import org.apache.hadoop.hdfs.server.namenode.INodeAttributes; +import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; +import org.apache.hadoop.hdfs.util.ReadOnlyList; +import org.apache.hadoop.ipc.CallerContext; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; +import org.devlive.grantforge.agent.AccessEvent; +import org.devlive.grantforge.agent.AgentDecision; +import org.devlive.grantforge.agent.Snapshot; +import org.devlive.grantforge.policy.engine.AccessRequest; +import org.jspecify.annotations.Nullable; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.springframework.test.util.ReflectionTestUtils; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import static java.util.Objects.requireNonNull; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +class HdfsAccessControlEnforcerTest +{ + private final List requests = new ArrayList<>(); + private final List events = new ArrayList<>(); + + static AgentDecision decision(String outcome) + { + AgentDecision decision = mock(AgentDecision.class); + AgentDecision.Outcome state = AgentDecision.Outcome.valueOf(outcome); + when(decision.outcome()).thenReturn(state); + when(decision.allowed()).thenReturn(state == AgentDecision.Outcome.ALLOWED); + when(decision.determined()).thenReturn(state != AgentDecision.Outcome.NOT_DETERMINED); + when(decision.policyId()).thenReturn(state == AgentDecision.Outcome.NOT_DETERMINED ? null : 42L); + when(decision.policyVersion()).thenReturn(7L); + return decision; + } + + static AuthorizationContext context(@Nullable FsAction action) + { + INodeDirectory root = directory(""); + INodeDirectory data = directory("data"); + INode file = file("file"); + return new AuthorizationContext.Builder().fsOwner("hdfs").supergroup("supergroup") + .callerUgi(UserGroupInformation.createUserForTesting("alice", new String[] {"analysts"})) + .inodeAttrs(new INodeAttributes[] {attributes("bob", (short) 0777), attributes("bob", (short) 0777), + attributes("bob", (short) 0666)}) + .inodes(new INode[] {root, data, file}).pathByNameArr(components("", "data", "file")) + .path("/data/file").snapshotId(19).ancestorIndex(1).access(action) + .operationName("open").callerContext(new CallerContext.Builder("client-request").build()).build(); + } + + private static byte[][] components(String... names) + { + byte[][] components = new byte[names.length][]; + for (int index = 0; index < names.length; index++) { + components[index] = names[index].getBytes(StandardCharsets.UTF_8); + } + return components; + } + + private static INodeAttributes attributes(String owner, short permission) + { + INodeAttributes attributes = mock(INodeAttributes.class); + when(attributes.getUserName()).thenReturn(owner); + when(attributes.getGroupName()).thenReturn("analysts"); + when(attributes.getFsPermission()).thenReturn(new FsPermission(permission)); + return attributes; + } + + private static INodeDirectory directory(String name) + { + INodeDirectory directory = mock(INodeDirectory.class); + when(directory.isDirectory()).thenReturn(true); + when(directory.asDirectory()).thenReturn(directory); + when(directory.getLocalNameBytes()).thenReturn(name.getBytes(StandardCharsets.UTF_8)); + when(directory.getChildrenList(anyInt())).thenReturn(ReadOnlyList.Util.emptyList()); + return directory; + } + + private static INode file(String name) + { + INode file = mock(INode.class); + when(file.getLocalNameBytes()).thenReturn(name.getBytes(StandardCharsets.UTF_8)); + return file; + } + + private HdfsAccessControlEnforcer enforcer(@Nullable AccessControlEnforcer nativeChecks, Function decide, + boolean fallback) + { + return new HdfsAccessControlEnforcer(nativeChecks, request -> { + requests.add(request); + return decide.apply(request); + }, events::add, () -> fallback); + } + + private HdfsAccessControlEnforcer allowing(@Nullable AccessControlEnforcer nativeChecks) + { + return enforcer(nativeChecks, request -> decision("ALLOWED"), false); + } + + private List checks() + { + return requests.stream().map(request -> request.resource().get("path") + ":" + request.accessType()).toList(); + } + + private static @Nullable Object field(AccessEvent event, String name) + { + return ReflectionTestUtils.getField(event, name); + } + + @ParameterizedTest + @EnumSource(FsAction.class) + void checksEveryRequestedActionBitAsWellAsTraversal(FsAction action) throws AccessControlException + { + AuthorizationContext context = context(action); + AccessControlEnforcer nativeChecks = mock(AccessControlEnforcer.class); + allowing(nativeChecks).checkPermissionWithContext(context); + + verify(nativeChecks).checkPermissionWithContext(context); + assertThat(checks()).contains("/:execute", "/data:execute"); + for (String type : List.of("read", "write", "execute")) { + FsAction bit = switch (type) { + case "read" -> FsAction.READ; + case "write" -> FsAction.WRITE; + default -> FsAction.EXECUTE; + }; + boolean expected = action.implies(bit); + assertThat(checks().contains("/data/file:" + type)).isEqualTo(expected); + } + assertThat(requests.get(0).user()).isEqualTo("alice"); + assertThat(requests.get(0).groups()).containsExactly("analysts"); + assertThat(requests.get(0).context()).containsEntry("operation", "open"); + assertThat(events).hasSameSizeAs(requests); + assertThat(field(events.get(0), "byGrantForge")).isEqualTo(true); + assertThat(field(events.get(0), "policyId")).isEqualTo(42L); + assertThat(field(events.get(0), "policyVersion")).isEqualTo(7L); + assertThat(field(events.get(0), "request")).isEqualTo("client-request"); + } + + @Test + void deniesACombinedPermissionWhenAnyOneBitIsDenied() + { + HdfsAccessControlEnforcer enforcer = enforcer(mock(AccessControlEnforcer.class), request -> + decision(request.accessType().equals("write") ? "DENIED" : "ALLOWED"), true); + + assertThatThrownBy(() -> enforcer.checkPermissionWithContext(context(FsAction.READ_WRITE))) + .isInstanceOf(AccessControlException.class).hasMessageContaining("write"); + assertThat(events).hasSize(1); + assertThat(field(events.get(0), "allowed")).isEqualTo(false); + assertThat(field(events.get(0), "resource")).isEqualTo("/data/file"); + assertThat(field(events.get(0), "accessType")).isEqualTo("write"); + } + + @Test + void doesNotFallBackOnExplicitDenyAndOnlyFallsBackOnUndeterminedAfterNativeChecks() throws AccessControlException + { + AccessControlEnforcer nativeChecks = mock(AccessControlEnforcer.class); + HdfsAccessControlEnforcer strict = enforcer(nativeChecks, request -> decision("NOT_DETERMINED"), false); + assertThatThrownBy(() -> strict.checkPermissionWithContext(context(FsAction.READ))) + .isInstanceOf(AccessControlException.class).hasMessageContaining("NOT_DETERMINED"); + events.clear(); + HdfsAccessControlEnforcer fallback = enforcer(nativeChecks, request -> decision("NOT_DETERMINED"), true); + fallback.checkPermissionWithContext(context(FsAction.READ)); + assertThat(field(events.get(0), "byGrantForge")).isEqualTo(false); + assertThat(field(events.get(0), "allowed")).isEqualTo(true); + HdfsAccessControlEnforcer denied = enforcer(nativeChecks, request -> decision("DENIED"), true); + assertThatThrownBy(() -> denied.checkPermissionWithContext(context(FsAction.READ))) + .isInstanceOf(AccessControlException.class).hasMessageContaining("DENIED"); + } + + @Test + void checksParentAndExistingAncestorAndTheMissingCreateTarget() throws AccessControlException + { + AuthorizationContext context = context(null); + context.setInodes(new INode[] {directory(""), directory("data"), null, null}); + context.setInodeAttrs(new INodeAttributes[] {attributes("bob", (short) 0777), attributes("bob", (short) 0777), null, null}); + context.setPathByNameArr(components("", "data", "new", "file")); + context.setPath("/data/new/file"); + context.setAncestorIndex(2); + context.setAncestorAccess(FsAction.WRITE); + context.setParentAccess(FsAction.WRITE); + allowing(mock(AccessControlEnforcer.class)).checkPermissionWithContext(context); + + assertThat(checks()).containsExactly("/:execute", "/data:execute", "/data:write", "/data/new/file:write"); + requests.clear(); + AuthorizationContext existing = context(null); + existing.setParentAccess(FsAction.WRITE_EXECUTE); + allowing(mock(AccessControlEnforcer.class)).checkPermissionWithContext(existing); + assertThat(checks()).contains("/data:write", "/data:execute", "/data/file:write", "/data/file:execute"); + } + + @Test + void enforcesOwnerOnlyOperationsAsWriteAndAllowsPureCreateTraversalWithoutTargetRead() throws AccessControlException + { + AuthorizationContext owner = context(null); + owner.setDoCheckOwner(true); + allowing(mock(AccessControlEnforcer.class)).checkPermissionWithContext(owner); + assertThat(checks()).contains("/data/file:write").doesNotContain("/data/file:read"); + requests.clear(); + AuthorizationContext createTraversal = context(null); + createTraversal.setOperationName("create"); + enforcer(mock(AccessControlEnforcer.class), request -> decision( + "execute".equals(request.accessType()) ? "ALLOWED" : "DENIED"), false) + .checkPermissionWithContext(createTraversal); + assertThat(checks()).containsExactly("/:execute", "/data:execute").doesNotContain("/data/file:read"); + } + + @Test + void handlesSingleInodeChecksAndNativeDenialWithNullPath() throws AccessControlException + { + AuthorizationContext context = context(FsAction.READ); + context.setInodes(new INode[] {context.getInodes()[2]}); + context.setInodeAttrs(new INodeAttributes[] {context.getInodeAttrs()[2]}); + context.setAncestorIndex(-1); + context.setPath(null); + allowing(mock(AccessControlEnforcer.class)).checkPermissionWithContext(context); + assertThat(checks()).containsExactly("/data/file:read"); + AccessControlEnforcer nativeChecks = mock(AccessControlEnforcer.class); + doThrow(new AccessControlException("native refused")).when(nativeChecks).checkPermissionWithContext(any()); + assertThatThrownBy(() -> allowing(nativeChecks).checkPermissionWithContext(context)).isInstanceOf(AccessControlException.class); + assertThat(field(events.get(events.size() - 1), "resource")).isEqualTo("/data/file"); + } + + @Test + void handlesRootOnlyNullComponent() throws AccessControlException + { + AuthorizationContext context = context(FsAction.READ_EXECUTE); + context.setInodes(new INode[] {context.getInodes()[0]}); + context.setInodeAttrs(new INodeAttributes[] {context.getInodeAttrs()[0]}); + context.setPathByNameArr(new byte[][] {null}); + context.setAncestorIndex(-1); + context.setPath(null); + allowing(mock(AccessControlEnforcer.class)).checkPermissionWithContext(context); + assertThat(checks()).containsExactly("/:read", "/:execute"); + } + + @Test + void rootNoneStillRequiresAPolicyAndMalformedEmptyContextFailsClosed() + { + AuthorizationContext context = context(FsAction.NONE); + context.setInodes(new INode[] {context.getInodes()[0]}); + context.setInodeAttrs(new INodeAttributes[] {context.getInodeAttrs()[0]}); + context.setPathByNameArr(new byte[][] {null}); + context.setAncestorIndex(-1); + context.setPath("/"); + assertThatThrownBy(() -> enforcer(mock(AccessControlEnforcer.class), request -> decision("NOT_DETERMINED"), false) + .checkPermissionWithContext(context)).isInstanceOf(AccessControlException.class); + assertThat(checks()).containsExactly("/:execute"); + context.setInodes(new INode[0]); + assertThatThrownBy(() -> allowing(mock(AccessControlEnforcer.class)).checkPermissionWithContext(context)) + .isInstanceOf(AccessControlException.class).hasMessageContaining("no inode path"); + } + + @Test + @SuppressWarnings("deprecation") + void passesEveryLegacyArgumentUnchangedAndAppliesTheSamePolicyChecks() throws AccessControlException + { + AuthorizationContext context = context(FsAction.READ); + context.setDoCheckOwner(true); + context.setAncestorAccess(FsAction.WRITE); + context.setParentAccess(FsAction.WRITE_EXECUTE); + context.setSubAccess(FsAction.READ_EXECUTE); + context.setIgnoreEmptyDir(true); + AccessControlEnforcer nativeChecks = mock(AccessControlEnforcer.class); + allowing(nativeChecks).checkPermission(context.getFsOwner(), context.getSupergroup(), context.getCallerUgi(), + context.getInodeAttrs(), context.getInodes(), context.getPathByNameArr(), context.getSnapshotId(), context.getPath(), + context.getAncestorIndex(), context.isDoCheckOwner(), context.getAncestorAccess(), context.getParentAccess(), + context.getAccess(), context.getSubAccess(), context.isIgnoreEmptyDir()); + verify(nativeChecks).checkPermission(context.getFsOwner(), context.getSupergroup(), context.getCallerUgi(), + context.getInodeAttrs(), context.getInodes(), context.getPathByNameArr(), context.getSnapshotId(), context.getPath(), + context.getAncestorIndex(), context.isDoCheckOwner(), context.getAncestorAccess(), context.getParentAccess(), + context.getAccess(), context.getSubAccess(), context.isIgnoreEmptyDir()); + assertThat(checks()).contains("/data:write", "/data/file:write", "/data/file:read", "/data/file:execute"); + } + + @Test + void checksEverySubtreeEntryWithTheSnapshotIdIncludingEmptyDirectoriesAndFiles() throws AccessControlException + { + INodeDirectory directory = directory("private"); + INodeDirectory empty = directory("empty"); + INode file = file("secret"); + when(directory.getChildrenList(19)).thenReturn(ReadOnlyList.Util.asReadOnlyList(List.of(empty, file))); + AuthorizationContext context = context(null); + context.getInodes()[2] = directory; + context.setPath("/data/private"); + context.setPathByNameArr(components("", "data", "private")); + context.setSubAccess(FsAction.ALL); + context.setIgnoreEmptyDir(true); + AccessControlEnforcer nativeChecks = mock(AccessControlEnforcer.class); + allowing(nativeChecks).checkPermissionWithContext(context); + + verify(nativeChecks).checkPermissionWithContext(context); + verify(directory).getChildrenList(19); + verify(empty).getChildrenList(19); + for (String path : List.of("/data/private", "/data/private/empty", "/data/private/secret")) { + assertThat(checks()).contains(path + ":read", path + ":write", path + ":execute"); + } + requests.clear(); + HdfsAccessControlEnforcer denied = enforcer(nativeChecks, request -> decision( + requireNonNull(request.resource().get("path")).endsWith("/secret") ? "DENIED" : "ALLOWED"), true); + assertThatThrownBy(() -> denied.checkPermissionWithContext(context)).isInstanceOf(AccessControlException.class) + .hasMessageContaining("/data/private/secret"); + } + + @Test + void boundsSubtreeWorkBeforeAllocatingPermissionChecks() + { + INodeDirectory directory = directory("large"); + INode child = file("child"); + when(directory.getChildrenList(19)).thenReturn(ReadOnlyList.Util.asReadOnlyList(Collections.nCopies(100_001, child))); + AuthorizationContext context = context(null); + context.getInodes()[2] = directory; + context.setSubAccess(FsAction.ALL); + assertThatThrownBy(() -> allowing(mock(AccessControlEnforcer.class)).checkPermissionWithContext(context)) + .isInstanceOf(AccessControlException.class).hasMessageContaining("subtree exceeds 100000"); + assertThat(requests).isEmpty(); + assertThat(field(events.get(0), "byGrantForge")).isEqualTo(true); + } + + @Test + void nativeTraversalOwnerStickyBitAndSubtreeDenialsCannotBeOverriddenByAnAllowPolicy() + { + NativeChecker nativeChecks = new NativeChecker(); + AuthorizationContext traversal = context(FsAction.READ); + traversal.getInodeAttrs()[0] = attributes("bob", (short) 0000); + assertNativeDenied(nativeChecks, traversal); + AuthorizationContext owner = context(FsAction.READ); + owner.setDoCheckOwner(true); + assertNativeDenied(nativeChecks, owner); + AuthorizationContext sticky = context(null); + sticky.setParentAccess(FsAction.WRITE); + sticky.getInodeAttrs()[1] = attributes("bob", (short) 01777); + assertNativeDenied(nativeChecks, sticky); + INodeDirectory restricted = directory("restricted"); + org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes restrictedAttributes = directoryAttributes("bob", (short) 0000); + when(restricted.getSnapshotINode(19)).thenReturn(restrictedAttributes); + AuthorizationContext subtree = context(null); + subtree.getInodes()[2] = restricted; + subtree.setSubAccess(FsAction.ALL); + assertNativeDenied(nativeChecks, subtree); + } + + @Test + void trueNativeReadSucceedsWhileNativeModeAndAclDenyOverrideAnAllowPolicy() throws AccessControlException + { + NativeChecker nativeChecks = new NativeChecker(); + allowing(nativeChecks).checkPermissionWithContext(context(FsAction.READ)); + assertThat(checks()).contains("/data/file:read"); + AuthorizationContext noRead = context(FsAction.READ); + noRead.getInodeAttrs()[2] = attributes("bob", (short) 0000); + assertNativeDenied(nativeChecks, noRead); + AuthorizationContext aclDeny = context(FsAction.READ); + AclEntry alice = new AclEntry.Builder().setType(AclEntryType.USER).setScope(AclEntryScope.ACCESS) + .setName("alice").setPermission(FsAction.NONE).build(); + when(aclDeny.getInodeAttrs()[2].getAclFeature()).thenReturn(new AclFeature(AclEntryStatusFormat.toInt(List.of(alice)))); + assertNativeDenied(nativeChecks, aclDeny); + } + + private static org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes directoryAttributes(String owner, short mode) + { + org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes attributes = + mock(org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes.class); + when(attributes.getUserName()).thenReturn(owner); + when(attributes.getGroupName()).thenReturn("analysts"); + when(attributes.getFsPermission()).thenReturn(new FsPermission(mode)); + return attributes; + } + + private void assertNativeDenied(AccessControlEnforcer nativeChecks, AuthorizationContext context) + { + requests.clear(); + assertThatThrownBy(() -> allowing(nativeChecks).checkPermissionWithContext(context)).isInstanceOf(AccessControlException.class); + assertThat(requests).isEmpty(); + assertThat(field(events.get(events.size() - 1), "allowed")).isEqualTo(false); + assertThat(field(events.get(events.size() - 1), "byGrantForge")).isEqualTo(false); + } + + @Test + void respectsNativeSuperuserPrivilegeAndConservativelyChecksPathPermissions() throws AccessControlException + { + AuthorizationContext context = context(null); + assertThatThrownBy(() -> allowing(new NativeChecker()).checkSuperUserPermissionWithContext(context)) + .isInstanceOf(AccessControlException.class).hasMessageContaining("Superuser"); + assertThat(requests).isEmpty(); + context.setCallerUgi(UserGroupInformation.createUserForTesting("hdfs", new String[] {"supergroup"})); + allowing(new NativeChecker()).checkSuperUserPermissionWithContext(context); + assertThat(checks()).containsExactly("/data/file:read", "/data/file:write", "/data/file:execute"); + requests.clear(); + context.setPath(null); + allowing(new NativeChecker()).checkSuperUserPermissionWithContext(context); + assertThat(requests).isEmpty(); + assertThat(field(events.get(events.size() - 1), "byGrantForge")).isEqualTo(false); + } + + @Test + void denyCallbackAlwaysDeniesAndAuditsAndPolicyEvaluationFailureFailsClosed() + { + AccessControlEnforcer nativeChecks = mock(AccessControlEnforcer.class); + HdfsAccessControlEnforcer enforcer = allowing(nativeChecks); + assertThatThrownBy(() -> enforcer.denyUserAccess(context(FsAction.READ), "not owner")) + .isInstanceOf(AccessControlException.class).hasMessage("not owner"); + verifyNoInteractions(nativeChecks); + assertThat(requests).isEmpty(); + assertThat(field(events.get(0), "allowed")).isEqualTo(false); + HdfsAccessControlEnforcer broken = enforcer(nativeChecks, request -> { + throw new IllegalStateException("broken engine"); + }, true); + assertThatThrownBy(() -> broken.checkPermissionWithContext(context(FsAction.READ))) + .isInstanceOf(AccessControlException.class).hasCauseInstanceOf(IllegalStateException.class); + assertThatThrownBy(() -> allowing(null).checkPermissionWithContext(context(FsAction.READ))) + .isInstanceOf(AccessControlException.class).hasMessageContaining("native permission"); + } + + @Test + void doesNotChangeDecisionsWhenAuditQueueThrowsAndBoundsOnlyTheAuditResource() throws AccessControlException + { + HdfsAccessControlEnforcer enforcer = new HdfsAccessControlEnforcer(mock(AccessControlEnforcer.class), + request -> decision("ALLOWED"), event -> { throw new IllegalStateException("audit failed"); }, () -> false); + enforcer.checkPermissionWithContext(context(FsAction.READ)); + AuthorizationContext context = context(FsAction.READ); + context.setPath("/" + "a".repeat(1200)); + allowing(mock(AccessControlEnforcer.class)).checkPermissionWithContext(context); + assertThat(requests.get(requests.size() - 1).resource().get("path")).hasSize(1201); + AccessEvent event = events.get(events.size() - 1); + assertThat((String) field(event, "resource")).hasSizeLessThanOrEqualTo(1000).endsWith("...[truncated]"); + assertThat((String) field(event, "request")).contains("resourceLength=1201", "resourceSha256="); + } + + @Test + void realPathPolicyCanDenyAChildWithoutDenyingItsParent() + { + Snapshot snapshot = Snapshot.parse(""" + {"format":1,"service":"cluster","serviceType":"hdfs","serviceEnabled":true,"policyVersion":1, + "definition":{"resources":[{"name":"path","parent":null,"matcher":"PATH","caseSensitive":true}], + "accessTypes":[{"name":"read","impliedGrants":[]},{"name":"write","impliedGrants":[]}, + {"name":"execute","impliedGrants":[]}],"conditions":[]}, + "policies":[{"id":"1","type":"ACCESS","name":"protected file","priority":"NORMAL","document":{ + "resources":{"path":{"values":["/data/private/secret"],"excludes":false,"recursive":false}}, + "deny":[{"users":["alice"],"groups":[],"roles":[],"accessTypes":["write"]}]}}], + "roles":{},"groups":{}} + """.getBytes(StandardCharsets.UTF_8), Map.of()); + INodeDirectory directory = directory("private"); + INode secret = file("secret"); + when(directory.getChildrenList(19)).thenReturn(ReadOnlyList.Util.asReadOnlyList(List.of(secret))); + AuthorizationContext context = context(null); + context.getInodes()[2] = directory; + context.setPath("/data/private"); + context.setSubAccess(FsAction.ALL); + + assertThatThrownBy(() -> enforcer(mock(AccessControlEnforcer.class), snapshot::decide, true) + .checkPermissionWithContext(context)).isInstanceOf(AccessControlException.class) + .hasMessageContaining("write").hasMessageContaining("/data/private/secret"); + } + + @Test + void snapshotsAlsoCheckTheLivePathAndCannotBypassAnExactFileDeny() throws AccessControlException + { + Snapshot snapshot = Snapshot.parse(""" + {"format":1,"service":"cluster","serviceType":"hdfs","serviceEnabled":true,"policyVersion":1, + "definition":{"resources":[{"name":"path","parent":null,"matcher":"PATH","caseSensitive":true}], + "accessTypes":[{"name":"read","impliedGrants":[]},{"name":"write","impliedGrants":[]}, + {"name":"execute","impliedGrants":[]}],"conditions":[]}, + "policies":[{"id":"1","type":"ACCESS","name":"broad allow","priority":"NORMAL","document":{ + "resources":{"path":{"values":["/"],"excludes":false,"recursive":true}}, + "allow":[{"users":["alice"],"groups":[],"roles":[],"accessTypes":["read","execute"]}]}}, + {"id":"2","type":"ACCESS","name":"exact deny","priority":"NORMAL","document":{ + "resources":{"path":{"values":["/data/file"],"excludes":false,"recursive":false}}, + "deny":[{"users":["alice"],"groups":[],"roles":[],"accessTypes":["read"]}]}}], + "roles":{},"groups":{}} + """.getBytes(StandardCharsets.UTF_8), Map.of()); + AuthorizationContext context = context(FsAction.READ); + context.setPath("/data/.snapshot/s1/file"); + context.setPathByNameArr(components("", "data", ".snapshot/s1", "file")); + context.setInodes(new INode[] {directory(""), directory("data"), directory("data"), file("file")}); + context.setInodeAttrs(new INodeAttributes[] {attributes("bob", (short) 0777), attributes("bob", (short) 0777), + attributes("bob", (short) 0777), attributes("bob", (short) 0666)}); + context.setAncestorIndex(2); + assertThatThrownBy(() -> enforcer(mock(AccessControlEnforcer.class), snapshot::decide, false) + .checkPermissionWithContext(context)).isInstanceOf(AccessControlException.class).hasMessageContaining("/data/file"); + assertThat(checks()).contains("/data/.snapshot/s1/file:read", "/data/file:read", "/data/.snapshot/s1:execute"); + requests.clear(); + allowing(mock(AccessControlEnforcer.class)).checkPermissionWithContext(context); + assertThat(checks()).contains("/data/.snapshot/s1/file:read", "/data/file:read"); + requests.clear(); + context.setPath("/data/.snapshot"); + allowing(mock(AccessControlEnforcer.class)).checkPermissionWithContext(context); + assertThat(checks()).contains("/data/.snapshot:read", "/data:read"); + } + + private static final class NativeChecker + extends FSPermissionChecker + { + NativeChecker() + { + super("hdfs", "supergroup", UserGroupInformation.createUserForTesting("alice", new String[] {"analysts"}), null); + } + } +} diff --git a/plugins/grantforge-agent-hdfs/src/test/java/org/devlive/grantforge/hdfs/agent/HdfsAgentSettingsTest.java b/plugins/grantforge-agent-hdfs/src/test/java/org/devlive/grantforge/hdfs/agent/HdfsAgentSettingsTest.java new file mode 100644 index 00000000..4bf02a46 --- /dev/null +++ b/plugins/grantforge-agent-hdfs/src/test/java/org/devlive/grantforge/hdfs/agent/HdfsAgentSettingsTest.java @@ -0,0 +1,121 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.hdfs.agent; + +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.util.Base64; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class HdfsAgentSettingsTest +{ + @TempDir + Path temporary; + + static Configuration configuration(Path temporary) throws IOException + { + Path token = temporary.resolve("token"); + Files.writeString(token, " secret-agent-token\n"); + Configuration configuration = new Configuration(false); + configuration.set("grantforge.hdfs.server.url", "https://grantforge.example.com/"); + configuration.set("grantforge.hdfs.token.file", token.toString()); + configuration.set("grantforge.hdfs.instance", "namenode-1:8020"); + configuration.set("grantforge.hdfs.cache.dir", temporary.resolve("cache").toString()); + return configuration; + } + + @Test + void readsRequiredSettingsAndDeniesUndeterminedAccessByDefault() throws IOException + { + HdfsAgentSettings settings = HdfsAgentSettings.read(configuration(temporary)); + + assertThat(settings.nativeFallback()).isFalse(); + assertThat(settings.agent().token()).isEqualTo("secret-agent-token"); + assertThat(settings.agent().instance()).isEqualTo("namenode-1:8020"); + assertThat(settings.agent().server().getScheme()).isEqualTo("https"); + assertThat(settings.agent().cacheDirectory()).isEqualTo(temporary.resolve("cache")); + assertThat(settings.agent().connectTimeout()).isEqualTo(Duration.ofSeconds(5)); + assertThat(settings.agent().readTimeout()).isEqualTo(Duration.ofSeconds(8)); + assertThat(settings.agent().refreshInterval()).isEqualTo(Duration.ofSeconds(30)); + } + + @Test + void readsExplicitFallbackTimeoutsAndPinnedSigningKey() throws IOException, NoSuchAlgorithmException + { + Configuration configuration = configuration(temporary); + configuration.set("grantforge.hdfs.native.fallback", "true"); + configuration.setLong("grantforge.hdfs.connect.timeout.ms", 400); + configuration.setLong("grantforge.hdfs.read.timeout.ms", 600); + configuration.setLong("grantforge.hdfs.refresh.interval.ms", 2000); + Path keyFile = temporary.resolve("signing-key"); + byte[] encoded = KeyPairGenerator.getInstance("Ed25519").generateKeyPair().getPublic().getEncoded(); + Files.writeString(keyFile, Base64.getEncoder().encodeToString(encoded)); + configuration.set("grantforge.hdfs.signing.key.file", keyFile.toString()); + + HdfsAgentSettings settings = HdfsAgentSettings.read(configuration); + + assertThat(settings.nativeFallback()).isTrue(); + assertThat(settings.agent().connectTimeout()).isEqualTo(Duration.ofMillis(400)); + assertThat(settings.agent().readTimeout()).isEqualTo(Duration.ofMillis(600)); + assertThat(settings.agent().refreshInterval()).isEqualTo(Duration.ofSeconds(2)); + assertThat(settings.agent().trustedKey()).isNotNull(); + } + + @Test + void rejectsMissingSettingsUnreadableOrEmptyTokenAndInvalidValues() throws IOException + { + assertThatThrownBy(() -> HdfsAgentSettings.read(new Configuration(false))).isInstanceOf(IllegalArgumentException.class); + Configuration configuration = configuration(temporary); + configuration.set("grantforge.hdfs.token.file", temporary.resolve("missing").toString()); + assertThatThrownBy(() -> HdfsAgentSettings.read(configuration)).isInstanceOf(IOException.class); + configuration.set("grantforge.hdfs.token.file", temporary.resolve("token").toString()); + Files.writeString(temporary.resolve("token"), "\n"); + assertThatThrownBy(() -> HdfsAgentSettings.read(configuration)).isInstanceOf(IllegalArgumentException.class); + Files.writeString(temporary.resolve("token"), "token"); + configuration.set("grantforge.hdfs.native.fallback", "typo"); + assertThatThrownBy(() -> HdfsAgentSettings.read(configuration)).isInstanceOf(IllegalArgumentException.class); + configuration.set("grantforge.hdfs.native.fallback", "false"); + configuration.setLong("grantforge.hdfs.refresh.interval.ms", 999); + assertThatThrownBy(() -> HdfsAgentSettings.read(configuration)).isInstanceOf(IllegalArgumentException.class); + configuration.setLong("grantforge.hdfs.refresh.interval.ms", 1000); + configuration.setLong("grantforge.hdfs.connect.timeout.ms", 0); + assertThatThrownBy(() -> HdfsAgentSettings.read(configuration)).isInstanceOf(IllegalArgumentException.class); + configuration.setLong("grantforge.hdfs.connect.timeout.ms", 1000); + configuration.set("grantforge.hdfs.server.url", "file:/tmp/server"); + assertThatThrownBy(() -> HdfsAgentSettings.read(configuration)).isInstanceOf(IllegalArgumentException.class); + } + + @Test + void rejectsDisablingNativePermissionsOrBypassingTheAgentAndBoundsNetworkTimeouts() throws IOException + { + Configuration configuration = configuration(temporary); + configuration.setBoolean("dfs.permissions.enabled", false); + assertThatThrownBy(() -> HdfsAgentSettings.read(configuration)).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("dfs.permissions.enabled"); + configuration.setBoolean("dfs.permissions.enabled", true); + configuration.set("dfs.namenode.inode.attributes.provider.bypass.users", "hdfs,alice"); + assertThatThrownBy(() -> HdfsAgentSettings.read(configuration)).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("bypass.users"); + configuration.unset("dfs.namenode.inode.attributes.provider.bypass.users"); + for (String key : new String[] {"grantforge.hdfs.connect.timeout.ms", "grantforge.hdfs.read.timeout.ms"}) { + configuration.setLong(key, Integer.MAX_VALUE); + assertThat(HdfsAgentSettings.read(configuration).agent()).isNotNull(); + configuration.setLong(key, Integer.MAX_VALUE + 1L); + assertThatThrownBy(() -> HdfsAgentSettings.read(configuration)).isInstanceOf(IllegalArgumentException.class); + configuration.unset(key); + } + } +} diff --git a/plugins/grantforge-agent-hdfs/src/test/java/org/devlive/grantforge/hdfs/agent/HdfsAuthorizationProviderTest.java b/plugins/grantforge-agent-hdfs/src/test/java/org/devlive/grantforge/hdfs/agent/HdfsAuthorizationProviderTest.java new file mode 100644 index 00000000..b3c7bb21 --- /dev/null +++ b/plugins/grantforge-agent-hdfs/src/test/java/org/devlive/grantforge/hdfs/agent/HdfsAuthorizationProviderTest.java @@ -0,0 +1,159 @@ +// Copyright (c) 2026 devlive-community/grantforge +// +// Licensed under the MIT License. See the LICENSE file in the +// project root for full license text. + +package org.devlive.grantforge.hdfs.agent; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer; +import org.apache.hadoop.hdfs.server.namenode.INodeAttributes; +import org.apache.hadoop.security.AccessControlException; +import org.devlive.grantforge.agent.AgentDecision; +import org.devlive.grantforge.agent.AgentSettings; +import org.devlive.grantforge.agent.GrantForgeAgent; +import org.devlive.grantforge.agent.Snapshot; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static java.util.Objects.requireNonNull; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class HdfsAuthorizationProviderTest +{ + @TempDir + Path temporary; + + @Test + void startsOnceStopsOnceAndCanRestartWithANewConfiguration() throws IOException + { + GrantForgeAgent agent = mock(GrantForgeAgent.class); + AtomicInteger starts = new AtomicInteger(); + AtomicReference received = new AtomicReference<>(); + HdfsAuthorizationProvider provider = new HdfsAuthorizationProvider(settings -> { + starts.incrementAndGet(); + received.set(settings); + return agent; + }); + Configuration configuration = HdfsAgentSettingsTest.configuration(temporary); + provider.setConf(configuration); + assertThat(provider.getConf()).isNotSameAs(configuration); + assertThat(requireNonNull(provider.getConf()).get("grantforge.hdfs.instance")).isEqualTo("namenode-1:8020"); + provider.start(); + provider.start(); + assertThat(starts).hasValue(1); + assertThat(requireNonNull(received.get()).instance()).isEqualTo("namenode-1:8020"); + assertThatThrownBy(() -> provider.setConf(new Configuration(false))).isInstanceOf(IllegalStateException.class); + provider.stop(); + provider.stop(); + verify(agent).close(); + provider.setConf(configuration); + provider.start(); + provider.stop(); + assertThat(starts).hasValue(2); + verify(agent, times(2)).close(); + } + + @Test + void configurationCopiesPreventExternalMutationOfAgentSettings() throws IOException + { + GrantForgeAgent agent = mock(GrantForgeAgent.class); + AtomicReference received = new AtomicReference<>(); + HdfsAuthorizationProvider provider = new HdfsAuthorizationProvider(settings -> { + received.set(settings); + return agent; + }); + assertThat(provider.getConf()).isNull(); + Configuration configuration = HdfsAgentSettingsTest.configuration(temporary); + provider.setConf(configuration); + configuration.set("grantforge.hdfs.instance", "mutated-input"); + Configuration returned = requireNonNull(provider.getConf()); + assertThat(returned.get("grantforge.hdfs.instance")).isEqualTo("namenode-1:8020"); + returned.set("grantforge.hdfs.instance", "mutated-getter"); + assertThat(requireNonNull(provider.getConf()).get("grantforge.hdfs.instance")).isEqualTo("namenode-1:8020"); + provider.start(); + assertThat(requireNonNull(received.get()).instance()).isEqualTo("namenode-1:8020"); + provider.stop(); + } + + @Test + void retainsNativeAttributesAndProvidesTheContextApiForHadoopsStartupProbe() + { + HdfsAuthorizationProvider provider = new HdfsAuthorizationProvider(); + INodeAttributes attributes = mock(INodeAttributes.class); + assertThat(provider.getAttributes(new String[] {"data"}, attributes)).isSameAs(attributes); + assertThat(provider.getExternalAccessControlEnforcer(null)).isInstanceOf(HdfsAccessControlEnforcer.class); + } + + @Test + void failsToStartWithoutValidConfiguration() throws IOException + { + HdfsAuthorizationProvider provider = new HdfsAuthorizationProvider(); + assertThatThrownBy(provider::start).isInstanceOf(IllegalStateException.class); + provider.setConf(new Configuration(false)); + assertThatThrownBy(provider::start).isInstanceOf(IllegalStateException.class); + Configuration configuration = HdfsAgentSettingsTest.configuration(temporary); + configuration.set("grantforge.hdfs.token.file", temporary.resolve("missing").toString()); + provider.setConf(configuration); + assertThatThrownBy(provider::start).isInstanceOf(IllegalStateException.class).hasCauseInstanceOf(IOException.class); + } + + @Test + void rejectsWrongServiceTypeEvenWhenNativeFallbackIsEnabled() throws IOException + { + GrantForgeAgent agent = mock(GrantForgeAgent.class); + Snapshot wrong = Snapshot.parse(""" + {"format":1,"service":"hive-service","serviceType":"hive","serviceEnabled":true,"policyVersion":1, + "definition":{"resources":[{"name":"path","parent":null,"matcher":"PATH","caseSensitive":true}], + "accessTypes":[{"name":"read","impliedGrants":[]}],"conditions":[]}, + "policies":[],"roles":{},"groups":{}} + """.getBytes(StandardCharsets.UTF_8), Map.of()); + when(agent.snapshot()).thenReturn(wrong); + HdfsAuthorizationProvider provider = new HdfsAuthorizationProvider(settings -> agent); + Configuration configuration = HdfsAgentSettingsTest.configuration(temporary); + configuration.setBoolean("grantforge.hdfs.native.fallback", true); + provider.setConf(configuration); + provider.start(); + + AccessControlEnforcer enforcer = provider.getExternalAccessControlEnforcer(mock(AccessControlEnforcer.class)); + assertThatThrownBy(() -> enforcer.checkPermissionWithContext(HdfsAccessControlEnforcerTest.context(FsAction.READ))) + .isInstanceOf(AccessControlException.class).hasMessageContaining("could not evaluate"); + verify(agent).record(any()); + provider.stop(); + } + + @Test + void rejectsPermissionChecksBeforeStartAndAfterStop() throws IOException + { + GrantForgeAgent agent = mock(GrantForgeAgent.class); + HdfsAuthorizationProvider provider = new HdfsAuthorizationProvider(settings -> agent); + AccessControlEnforcer enforcer = provider.getExternalAccessControlEnforcer(mock(AccessControlEnforcer.class)); + assertThatThrownBy(() -> enforcer.checkPermissionWithContext(HdfsAccessControlEnforcerTest.context(FsAction.READ))) + .isInstanceOf(AccessControlException.class); + provider.setConf(HdfsAgentSettingsTest.configuration(temporary)); + provider.start(); + Snapshot snapshot = mock(Snapshot.class); + when(snapshot.serviceType()).thenReturn("hdfs"); + AgentDecision allowed = HdfsAccessControlEnforcerTest.decision("ALLOWED"); + when(snapshot.decide(any())).thenReturn(allowed); + when(agent.snapshot()).thenReturn(snapshot); + enforcer.checkPermissionWithContext(HdfsAccessControlEnforcerTest.context(FsAction.READ)); + provider.stop(); + assertThatThrownBy(() -> enforcer.checkPermissionWithContext(HdfsAccessControlEnforcerTest.context(FsAction.READ))) + .isInstanceOf(AccessControlException.class); + } +} diff --git a/pom.xml b/pom.xml index b05a11b7..f9b1e45d 100644 --- a/pom.xml +++ b/pom.xml @@ -33,6 +33,7 @@ core/grantforge-oauth plugins/grantforge-plugin-hdfs + plugins/grantforge-agent-hdfs core/grantforge-server sdk/grantforge-spring-boot-starter plugins/grantforge-plugin-example From 362546ccb55a3001198edc210e3437d166867b9c Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 07:32:38 -0400 Subject: [PATCH 17/22] feat(release): cut releases with one script and list their commits script/release/tag.sh checks the tree and the branch, sets the version everywhere, tags v and pushes it, and can move the branch on to the next version; --dry-run only previews. The GitHub release notes are now every commit since the previous release, grouped by type with links, instead of a changelog page; AuthX's bare version tags such as 1.0.6 count as releases, marker tags do not. --- docs/content/architecture/development.md | 16 +- script/ci/release.sh | 6 +- script/ci/release_notes.py | 195 ++++++++++++++++------- script/ci/tests/test_release_notes.py | 141 +++++++++++----- script/release/tag.sh | 105 ++++++++++++ 5 files changed, 361 insertions(+), 102 deletions(-) create mode 100755 script/release/tag.sh diff --git a/docs/content/architecture/development.md b/docs/content/architecture/development.md index 58b88b60..c153f509 100644 --- a/docs/content/architecture/development.md +++ b/docs/content/architecture/development.md @@ -63,13 +63,23 @@ bash script/ci/perf_benchmark.sh smoke # 小规模性能基准 ## 发布 -版本号为 `年.次版本.修订`（如 `2026.0.0`），候选版加 `-rc.N`。所有 pom、npm 包、Helm Chart 的 appVersion、控制台侧栏与 README 中的版本必须一致，CI 用 `check_versions.py` 检查；改版本只需一条命令： +版本号为 `年.次版本.修订`（如 `2026.0.0`），候选版加 `-rc.N`。所有 pom、npm 包、Helm Chart 的 appVersion、控制台侧栏与 README 中的版本必须一致，CI 用 `check_versions.py` 检查。 + +在 `dev` 分支上用一条命令发布： ```bash -python3 script/ci/check_versions.py --set 2026.1.0 +bash script/release/tag.sh 2026.1.0 --dry-run # 只检查并预览发布说明，不做任何修改 +bash script/release/tag.sh 2026.1.0 --next 2026.2.0 # 设置版本、打标签 v2026.1.0 并推送，再把 dev 切到下一个版本 ``` -发布前先在 `docs/content/changelog/` 写好标题为该版本的页面，然后推送标签 `v<版本>`。`release.yml` 运行 `script/ci/release.sh`（本地可同样运行）：构建发行包、只含发行依赖的 CycloneDX SBOM 与 `SHA256SUMS`，以更新日志页面为发布说明创建 GitHub Release，并把多架构镜像推送到 GHCR。候选版标记为预发布，不更新镜像的 `latest`。 +脚本要求工作区干净、本地分支不落后于远端、标签不存在，确认后提交 `chore(release): prepare <版本>`、创建带注释的标签并推送。标签触发 `release.yml`： + +- `script/ci/release.sh` 构建发行包、只含发行依赖的 CycloneDX SBOM 与 `SHA256SUMS`； +- 多架构镜像推送到 `ghcr.io/devlive-community/grantforge`； +- Maven 构件（含源码包与 Javadoc）发布到 GitHub Packages；仓库配置了 `CENTRAL_USERNAME`、`CENTRAL_PASSWORD`（Central Portal 令牌）、`GPG_PRIVATE_KEY` 与 `GPG_PASSPHRASE` 时，签名后发布到 Maven Central； +- 创建 GitHub Release，正文是上一个发布版本（`v*` 或 `1.0.6` 这样的数字标签）以来的全部提交，按新功能、问题修复、性能等分组，附提交链接。 + +候选版标记为预发布，不更新镜像的 `latest`。本地启用 `central` profile 默认不会发布（`central.skip=true`），只有发布工作流显式传入 `-Dcentral.skip=false`。 ## 权限清单 diff --git a/script/ci/release.sh b/script/ci/release.sh index 3b92ec27..3ca668e2 100755 --- a/script/ci/release.sh +++ b/script/ci/release.sh @@ -8,8 +8,8 @@ # release notes. .github/workflows/release.yml runs it for a pushed tag, then publishes the image and the GitHub # release; run it locally the same way to see what a release would contain. # -# release.sh v2026.1.0 the tag must be "v" + the version in every pom and package (check_versions.py), and -# docs/content/changelog/ must have a page titled with the version +# release.sh v2026.1.0 the tag must be "v" + the version in every pom and package (check_versions.py); the notes +# list the commits since the previous release (release_notes.py), so the tag must exist # # Files: grantforge-.tar.gz, grantforge-.sbom.json (CycloneDX, shipped dependencies only), # SHA256SUMS and notes.md. Set GRANTFORGE_RELEASE_SKIP_TESTS=1 to skip the unit tests (CI already ran them). @@ -24,7 +24,7 @@ VERSION="${TAG#v}" OUT="${ROOT}/target/release" rm -rf "${OUT}" mkdir -p "${OUT}" -python3 script/ci/release_notes.py "${VERSION}" --output "${OUT}/notes.md" +python3 script/ci/release_notes.py "${TAG}" --output "${OUT}/notes.md" MVN=(./mvnw --batch-mode --no-transfer-progress) if [[ "${GRANTFORGE_RELEASE_SKIP_TESTS:-0}" == "1" ]]; then diff --git a/script/ci/release_notes.py b/script/ci/release_notes.py index f2e7d3c2..18e25b4b 100755 --- a/script/ci/release_notes.py +++ b/script/ci/release_notes.py @@ -4,88 +4,171 @@ # Licensed under the MIT License. See the LICENSE file in the # project root for full license text. -"""Write the release notes of a version from its page in the documentation site's changelog. +"""Write the release notes of a version from the commit history (D-90). -The notes are the page of ``docs/content/changelog/`` whose title starts with the -version (``2026.0.0`` or ``2026.0.0（重构版）``), without its front matter and -license comment, and with the site's own links made absolute so they work on the -GitHub release page. A release without such a page fails: the changelog is -written before the tag. +The notes list every commit since the previous release, grouped by its conventional +commit type, each with a link to the commit, after a link comparing the two tags. +Breaking changes come first; merge commits are left out. A release tag is ``v`` and a +version (``v2026.1.0``), or a bare version as AuthX tagged its releases (``1.0.6``); +marker tags such as ``legacy-2026.0.0`` are not releases. Without any earlier release +the whole history is listed. Usage:: - python3 script/ci/release_notes.py 2026.0.0 [--output FILE] [--root DIR] + python3 script/ci/release_notes.py v2026.1.0 [--head v2026.1.0] [--output FILE] [--repo OWNER/NAME] + +``--head`` defaults to the tag; ``script/release/tag.sh`` passes ``HEAD`` to preview +the notes of a tag it has not made yet. The repository defaults to +``$GITHUB_REPOSITORY``, then to the ``origin`` remote. """ from __future__ import annotations import argparse +import os import re +import subprocess import sys +from dataclasses import dataclass from pathlib import Path -from typing import Optional, Sequence +from typing import Dict, List, Optional, Sequence, Tuple DEFAULT_ROOT = Path(__file__).resolve().parents[2] -_FRONT_MATTER = re.compile(r"\A---\n(.*?)\n---\n", re.S) -_TITLE = re.compile(r"^title:\s*(.+?)\s*$", re.M) -_LICENSE = re.compile(r"\A\s*\s*", re.S) -# Markdown links to the site's own pages: ](/start/upgrade/) or ](/start/upgrade/#anchor). -_SITE_LINK = re.compile(r"\]\((/[^)\s]*)\)") - - -def page_title(text: str) -> Optional[str]: - """Return the title in a page's front matter, without quotes.""" - front = _FRONT_MATTER.match(text) - title = _TITLE.search(front.group(1)) if front else None - return title.group(1).strip("\"'") if title else None - - -def names_version(title: str, version: str) -> bool: - """Whether a changelog title is about the version: it is the version, or starts with it and no digit or dot.""" - return title == version or (title.startswith(version) and not re.match(r"[\d.]", title[len(version)])) - - -def find_page(root: Path, version: str) -> Optional[Path]: - """Return the changelog page of the version.""" - for page in sorted((root / "docs" / "content" / "changelog").glob("*.md")): - title = page_title(page.read_text(encoding="utf-8")) - if title and names_version(title, version): - return page - return None - - -def site_url(root: Path) -> str: - """Return the documentation site's address, from the domain it is published under.""" - domain = (root / "docs" / "public" / "CNAME").read_text(encoding="utf-8").strip() - return "https://" + domain - - -def notes(text: str, site: str) -> str: - """Return a page's body as release notes.""" - front = _FRONT_MATTER.match(text) - body = text[front.end():] if front else text - body = _LICENSE.sub("", body, count=1) - body = _SITE_LINK.sub(lambda match: "](" + site + match.group(1) + ")", body) - return body.strip() + "\n" +# GitHub refuses release bodies above 125000 characters. +MAX_BODY = 120_000 + +_HEADER = re.compile(r"^(?P[a-z]+)(?:\((?P[^)]+)\))?(?P!)?: (?P.+)$") +_REMOTE = re.compile(r"github\.com[:/](?P[^/\s]+/[^/\s]+?)(?:\.git)?$") + +# Sections in their order, with the commit types each holds; types not listed go to the last section. +SECTIONS: Tuple[Tuple[str, Tuple[str, ...]], ...] = ( + ("新功能", ("feat",)), + ("问题修复", ("fix",)), + ("性能", ("perf",)), + ("重构", ("refactor",)), + ("文档", ("docs",)), + ("测试", ("test",)), + ("构建与 CI", ("build", "ci")), + ("其他", ()), +) + + +@dataclass(frozen=True) +class Commit: + sha: str + header: str + body: str + + +def git(root: Path, *args: str) -> str: + result = subprocess.run(["git", *args], cwd=root, check=True, stdout=subprocess.PIPE) + return result.stdout.decode("utf-8").strip() + + +def previous_tag(root: Path, head: str) -> Optional[str]: + """Return the latest release tag reachable from the commit before ``head``, or None for the first release.""" + result = subprocess.run(["git", "describe", "--tags", "--abbrev=0", "--match", "v[0-9]*", "--match", "[0-9]*", + f"{head}^"], cwd=root, + check=False, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + return result.stdout.decode("utf-8").strip() or None if result.returncode == 0 else None + + +def commits(root: Path, previous: Optional[str], head: str) -> List[Commit]: + """Return the commits since ``previous`` (all of them without one), oldest first, merges left out.""" + revision = f"{previous}..{head}" if previous else head + # Raw output: str.strip() takes the \x1e and \x1f separators for white space. + command = ["git", "log", "--reverse", "--no-merges", "--format=%H%x1f%s%x1f%b%x1e", revision] + output = subprocess.run(command, cwd=root, check=True, stdout=subprocess.PIPE).stdout.decode("utf-8") + found: List[Commit] = [] + for record in output.split("\x1e"): + record = record.lstrip("\n") + if record.strip(): + sha, header, body = record.split("\x1f", 2) + found.append(Commit(sha, header, body)) + return found + + +def repository(root: Path, given: Optional[str]) -> str: + """Return ``owner/name`` of the GitHub repository.""" + if given: + return given + if os.environ.get("GITHUB_REPOSITORY"): + return os.environ["GITHUB_REPOSITORY"] + match = _REMOTE.search(git(root, "remote", "get-url", "origin")) + if not match: + raise ValueError("cannot tell the GitHub repository; pass --repo OWNER/NAME") + return match.group("repo") + + +def breaks(commit: Commit) -> bool: + """Whether a commit is marked as breaking, by ``!`` in its header or a BREAKING CHANGE footer.""" + match = _HEADER.match(commit.header) + return bool(match and match.group("breaking")) or "BREAKING CHANGE" in commit.body + + +def section_of(commit_type: str) -> str: + for title, types in SECTIONS: + if commit_type in types: + return title + return SECTIONS[-1][0] + + +def line(commit: Commit, repo: str) -> str: + match = _HEADER.match(commit.header) + text = commit.header + if match: + scope = match.group("scope") + text = f"**{scope}**: {match.group('subject')}" if scope else match.group("subject") + return f"- {text} ([{commit.sha[:8]}](https://github.com/{repo}/commit/{commit.sha}))" + + +def notes(found: Sequence[Commit], repo: str, previous: Optional[str], tag: str) -> str: + """Return the notes: the comparison link, breaking changes, then the commits by section.""" + parts: List[str] = [] + if previous: + parts.append(f"**完整变更**：[{previous}...{tag}](https://github.com/{repo}/compare/{previous}...{tag})" + f"，共 {len(found)} 个提交。") + else: + parts.append(f"首个发布版本，包含仓库的全部 {len(found)} 个提交。") + breaking = [commit for commit in found if breaks(commit)] + if breaking: + parts.append("## 不兼容变更\n\n" + "\n".join(line(commit, repo) for commit in breaking)) + grouped: Dict[str, List[str]] = {title: [] for title, _ in SECTIONS} + for commit in reversed(found): + match = _HEADER.match(commit.header) + grouped[section_of(match.group("type") if match else "")].append(line(commit, repo)) + for title, _ in SECTIONS: + if grouped[title]: + parts.append(f"## {title}\n\n" + "\n".join(grouped[title])) + body = "\n\n".join(parts) + "\n" + if len(body) > MAX_BODY: + cut = body.rfind("\n", 0, MAX_BODY - 200) + body = body[:cut] + f"\n\n……其余提交见 [提交历史](https://github.com/{repo}/commits/{tag})。\n" + return body def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("version", help="the version, such as 2026.0.0") + parser.add_argument("tag", help="the release tag, such as v2026.1.0") + parser.add_argument("--head", help="the commit the release is made of; the tag by default") parser.add_argument("--output", type=Path, help="file to write; standard output by default") + parser.add_argument("--repo", help="GitHub repository OWNER/NAME") parser.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="repository root") args = parser.parse_args(argv) root: Path = args.root.resolve() - page = find_page(root, args.version) - if page is None: - print(f"no page in docs/content/changelog/ is titled {args.version}; write the changelog before tagging", - file=sys.stderr) + head = args.head or args.tag + try: + repo = repository(root, args.repo) + previous = previous_tag(root, head) + written = notes(commits(root, previous, head), repo, previous, args.tag) + except (subprocess.CalledProcessError, ValueError) as failure: + print(f"cannot write the release notes of {args.tag}: {failure}", file=sys.stderr) return 1 - written = notes(page.read_text(encoding="utf-8"), site_url(root)) if args.output: args.output.write_text(written, encoding="utf-8") - print(f"release notes of {args.version} from {page.relative_to(root)} written to {args.output}") + since = f"since {previous}" if previous else "whole history" + print(f"release notes of {args.tag} ({since}) written to {args.output}") else: sys.stdout.write(written) return 0 diff --git a/script/ci/tests/test_release_notes.py b/script/ci/tests/test_release_notes.py index 5e3b2371..ed7c5f63 100644 --- a/script/ci/tests/test_release_notes.py +++ b/script/ci/tests/test_release_notes.py @@ -3,12 +3,13 @@ # Licensed under the MIT License. See the LICENSE file in the # project root for full license text. -"""Unit tests for script/ci/release_notes.py (stdlib unittest, no dependencies).""" +"""Unit tests for script/ci/release_notes.py (stdlib unittest, against a temporary git repository).""" from __future__ import annotations import importlib.util import shutil +import subprocess import sys import tempfile import unittest @@ -21,51 +22,111 @@ sys.modules["release_notes"] = notes _SPEC.loader.exec_module(notes) -_PAGE = """--- -title: {title} -description: What changed. ---- - - -The changes. See [the upgrade](/start/upgrade/) and [GitHub](https://github.com/x). -""" +REPO = "acme/grantforge" class ReleaseNotesTest(unittest.TestCase): def setUp(self) -> None: self.root = Path(tempfile.mkdtemp()) - self.addCleanup(shutil.rmtree, self.root) - changelog = self.root / "docs" / "content" / "changelog" - changelog.mkdir(parents=True) - (changelog / "rebuild.md").write_text(_PAGE.format(title="2026.0.0（重构版）"), encoding="utf-8") - (changelog / "1.0.6.md").write_text(_PAGE.format(title='"1.0.6"'), encoding="utf-8") - (self.root / "docs" / "public").mkdir(parents=True) - (self.root / "docs" / "public" / "CNAME").write_text("docs.example.org\n", encoding="utf-8") - - def test_finds_the_page_by_its_title(self) -> None: - self.assertEqual(notes.find_page(self.root, "2026.0.0").name, "rebuild.md") - self.assertEqual(notes.find_page(self.root, "1.0.6").name, "1.0.6.md") - self.assertIsNone(notes.find_page(self.root, "2026.0")) - self.assertIsNone(notes.find_page(self.root, "1.0")) - - def test_a_title_names_a_version_only_when_nothing_numeric_follows(self) -> None: - self.assertTrue(notes.names_version("2026.0.0", "2026.0.0")) - self.assertTrue(notes.names_version("2026.0.0 (rebuild)", "2026.0.0")) - self.assertFalse(notes.names_version("2026.0.01", "2026.0.0")) - self.assertFalse(notes.names_version("2026.0.0.1", "2026.0.0")) - - def test_notes_drop_the_header_and_make_site_links_absolute(self) -> None: - written = notes.notes(_PAGE.format(title="2026.0.0"), notes.site_url(self.root)) - self.assertEqual(written, "The changes. See [the upgrade](https://docs.example.org/start/upgrade/) and " - "[GitHub](https://github.com/x).\n") - - def test_main_writes_the_notes_or_fails_without_a_page(self) -> None: + self.addCleanup(shutil.rmtree, self.root, True) + self._git("init", "-q", "-b", "dev") + self._git("config", "user.name", "Tester") + self._git("config", "user.email", "tester@example.org") + + def _git(self, *args: str) -> str: + result = subprocess.run(["git", *args], cwd=self.root, check=True, stdout=subprocess.PIPE) + return result.stdout.decode("utf-8").strip() + + def _commit(self, message: str) -> str: + self._git("commit", "-q", "--allow-empty", "-m", message) + return self._git("rev-parse", "HEAD") + + def _main(self, *args: str) -> int: + return notes.main(["--root", str(self.root), "--repo", REPO, *args]) + + def test_the_first_release_lists_the_whole_history_by_type(self) -> None: + start = self._commit("chore: start") + feature = self._commit("feat(web): add the console") + self._commit("fix: repair the sign-in") + self._commit("Legacy message") + self._git("tag", "v2026.0.0") + + found = notes.commits(self.root, None, "v2026.0.0") + self.assertEqual([commit.sha for commit in found][:2], [start, feature]) + written = notes.notes(found, REPO, None, "v2026.0.0") + self.assertTrue(written.startswith("首个发布版本，包含仓库的全部 4 个提交。")) + link = f"[{feature[:8]}](https://github.com/{REPO}/commit/{feature})" + self.assertIn(f"## 新功能\n\n- **web**: add the console ({link})", written) + self.assertIn("## 问题修复\n\n- repair the sign-in", written) + self.assertIn("## 其他\n\n- Legacy message", written) + self.assertLess(written.index("## 新功能"), written.index("## 问题修复")) + self.assertNotIn("## 不兼容变更", written) + + def test_a_later_release_lists_what_came_after_the_previous_tag(self) -> None: + self._commit("feat: add one") + self._git("tag", "v2026.0.0") + self._git("tag", "legacy-2026.0.0") + self._commit("feat!: drop the old API") + self._commit("refactor: tidy up\n\nBREAKING CHANGE: the setting is renamed") + self._git("checkout", "-q", "-b", "topic") + self._commit("perf: speed up") + self._git("checkout", "-q", "dev") + self._git("merge", "-q", "--no-ff", "topic", "-m", "Merge branch 'topic'") + + self.assertEqual(notes.previous_tag(self.root, "HEAD"), "v2026.0.0") + written = notes.notes(notes.commits(self.root, "v2026.0.0", "HEAD"), REPO, "v2026.0.0", "v2026.1.0") + compare = f"https://github.com/{REPO}/compare/v2026.0.0...v2026.1.0" + self.assertTrue(written.startswith(f"**完整变更**：[v2026.0.0...v2026.1.0]({compare})，共 3 个提交。")) + breaking = written[written.index("## 不兼容变更"):written.index("## 新功能")] + self.assertIn("drop the old API", breaking) + self.assertIn("tidy up", breaking) + self.assertIn("## 性能\n\n- speed up", written) + self.assertNotIn("Merge branch", written) + self.assertNotIn("add one", written) + + def test_earlier_releases_tagged_without_v_count_but_marker_tags_do_not(self) -> None: + self._commit("chore: release 1.0.6") + self._git("tag", "1.0.6") + self._commit("feat: rebuild") + self._git("tag", "legacy-2026.0.0") + self._commit("fix: repair") + + self.assertEqual(notes.previous_tag(self.root, "HEAD"), "1.0.6") + written = notes.notes(notes.commits(self.root, "1.0.6", "HEAD"), REPO, "1.0.6", "v2026.0.0") + self.assertIn("[1.0.6...v2026.0.0]", written) + self.assertIn("共 2 个提交", written) + + def test_the_tag_itself_is_no_previous_release(self) -> None: + self._commit("feat: add one") + self._git("tag", "v2026.0.0") + self._commit("fix: repair") + self._git("tag", "v2026.0.1") + + self.assertEqual(notes.previous_tag(self.root, "v2026.0.1"), "v2026.0.0") + self.assertIsNone(notes.previous_tag(self.root, "v2026.0.0")) + + def test_huge_bodies_are_cut_with_a_link(self) -> None: + found = [notes.Commit(f"{index:040x}", "feat: " + "x" * 200, "") for index in range(800)] + written = notes.notes(found, REPO, None, "v2026.0.0") + self.assertLessEqual(len(written), notes.MAX_BODY) + self.assertTrue(written.rstrip().endswith(f"(https://github.com/{REPO}/commits/v2026.0.0)。")) + + def test_main_writes_the_notes_and_reports_failures(self) -> None: + self._commit("feat: add one") output = self.root / "notes.md" - self.assertEqual(notes.main(["2026.0.0", "--root", str(self.root), "--output", str(output)]), 0) - self.assertTrue(output.read_text(encoding="utf-8").startswith("The changes.")) - self.assertEqual(notes.main(["2027.0.0", "--root", str(self.root)]), 1) + self.assertEqual(self._main("v2026.0.0", "--head", "HEAD", "--output", str(output)), 0) + self.assertIn("- add one", output.read_text(encoding="utf-8")) + self.assertEqual(self._main("v2026.0.0"), 1) + + def test_the_repository_comes_from_the_origin_remote(self) -> None: + self._git("remote", "add", "origin", "git@github.com:devlive-community/grantforge.git") + self.assertEqual(notes.repository(self.root, None if "GITHUB_REPOSITORY" not in notes.os.environ else + notes.os.environ["GITHUB_REPOSITORY"]), notes.os.environ.get( + "GITHUB_REPOSITORY", "devlive-community/grantforge")) + self._git("remote", "set-url", "origin", "https://example.org/other.git") + if "GITHUB_REPOSITORY" not in notes.os.environ: + with self.assertRaises(ValueError): + notes.repository(self.root, None) if __name__ == "__main__": diff --git a/script/release/tag.sh b/script/release/tag.sh new file mode 100755 index 00000000..1b50ad0d --- /dev/null +++ b/script/release/tag.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Copyright (c) 2026 devlive-community/grantforge +# +# Licensed under the MIT License. See the LICENSE file in the +# project root for full license text. + +# Cut a release (D-90): set the version everywhere, tag v and push it. The tag starts +# .github/workflows/release.yml, which builds and publishes the distribution, the Docker image, the Maven artifacts +# (GitHub Packages, and Maven Central when its secrets are set) and the GitHub release, whose notes are the commits +# since the previous release. +# +# tag.sh 2026.1.0 release 2026.1.0 from the current branch (dev by default) +# tag.sh 2026.1.0 --next 2026.2.0 then set the version the branch continues with +# tag.sh 2026.1.0 --dry-run check everything and show the notes; change nothing +# +# Options: --remote NAME (origin), --branch NAME (dev; the branch releases are cut from), --yes (do not ask). +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "${ROOT}" + +usage() { + echo "usage: $0 [--next ] [--remote ] [--branch ] [--dry-run] [--yes]" >&2 + exit 2 +} + +[[ $# -ge 1 ]] || usage +VERSION="$1" +shift +NEXT="" +REMOTE="origin" +BRANCH="dev" +DRY_RUN=0 +ASSUME_YES=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --next) [[ $# -ge 2 ]] || usage; NEXT="$2"; shift 2 ;; + --remote) [[ $# -ge 2 ]] || usage; REMOTE="$2"; shift 2 ;; + --branch) [[ $# -ge 2 ]] || usage; BRANCH="$2"; shift 2 ;; + --dry-run) DRY_RUN=1; shift ;; + --yes) ASSUME_YES=1; shift ;; + *) usage ;; + esac +done +TAG="v${VERSION}" + +fail() { + echo "error: $*" >&2 + exit 1 +} + +pattern='^[0-9]{4}\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$' +[[ "${VERSION}" =~ ${pattern} ]] || fail "${VERSION} is not YEAR.MINOR.PATCH, optionally with -rc.N" +[[ -z "${NEXT}" || "${NEXT}" =~ ${pattern} ]] || fail "${NEXT} is not YEAR.MINOR.PATCH, optionally with -rc.N" +[[ -z "$(git status --porcelain)" ]] || fail "the working tree has changes; commit or stash them first" +CURRENT_BRANCH="$(git symbolic-ref --quiet --short HEAD || true)" +[[ "${CURRENT_BRANCH}" == "${BRANCH}" ]] || fail "releases are cut from ${BRANCH}, not ${CURRENT_BRANCH:-a detached HEAD} (--branch to change)" + +git fetch --quiet --tags "${REMOTE}" "${BRANCH}" +if ! git merge-base --is-ancestor "${REMOTE}/${BRANCH}" HEAD; then + fail "${BRANCH} is behind ${REMOTE}/${BRANCH}; pull first" +fi +if git rev-parse --quiet --verify "refs/tags/${TAG}" > /dev/null || [[ -n "$(git ls-remote --tags "${REMOTE}" "refs/tags/${TAG}")" ]]; then + fail "${TAG} exists already" +fi + +CURRENT_VERSION="$(python3 script/ci/check_versions.py --show)" +echo "Releasing ${TAG} from ${BRANCH} (version now ${CURRENT_VERSION})" +echo +python3 script/ci/release_notes.py "${TAG}" --head HEAD | head -40 +echo "..." +echo + +if [[ "${DRY_RUN}" == "1" ]]; then + echo "Dry run: nothing changed. Without --dry-run this would:" + [[ "${CURRENT_VERSION}" == "${VERSION}" ]] || echo " - set the version to ${VERSION} and commit it" + echo " - tag ${TAG} and push ${BRANCH} and ${TAG} to ${REMOTE}" + [[ -z "${NEXT}" ]] || echo " - set the version to ${NEXT}, commit and push it" + exit 0 +fi + +if [[ "${ASSUME_YES}" != "1" ]]; then + read -r -p "Tag ${TAG} and push it to ${REMOTE}, which publishes the release? [y/N] " answer + [[ "${answer}" == "y" || "${answer}" == "Y" ]] || fail "cancelled" +fi + +if [[ "${CURRENT_VERSION}" != "${VERSION}" ]]; then + python3 script/ci/check_versions.py --set "${VERSION}" + git commit --quiet --all --message "chore(release): prepare ${VERSION}" +fi +python3 script/ci/check_versions.py --tag "${TAG}" +git tag --annotate "${TAG}" --message "GrantForge ${VERSION}" +git push --quiet "${REMOTE}" "${BRANCH}" +git push --quiet "${REMOTE}" "${TAG}" +echo "Pushed ${TAG}; the release workflow builds and publishes it." + +if [[ -n "${NEXT}" ]]; then + python3 script/ci/check_versions.py --set "${NEXT}" + git commit --quiet --all --message "chore(release): start ${NEXT}" + git push --quiet "${REMOTE}" "${BRANCH}" + echo "${BRANCH} continues with ${NEXT}." +fi + +REPOSITORY="$(git remote get-url "${REMOTE}" | sed -E 's#^.*github\.com[:/]##; s#\.git$##')" +echo "Follow it at https://github.com/${REPOSITORY}/actions/workflows/release.yml" From 4b3b27c4dbd73b6bb93c90c5f00e4ec9c5417773 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 07:32:38 -0400 Subject: [PATCH 18/22] build: publish the Maven artifacts of a release The release profile adds sources, Javadoc and GPG signatures; the release workflow deploys them to GitHub Packages and, when the Central Portal token and the GPG key are set as secrets, to Maven Central through the central profile. That profile publishes only with -Dcentral.skip=false, so a local build with Central credentials in settings.xml uploads nothing. The example plugin, a test fixture, is never published. --- .github/workflows/release.yml | 46 ++++++++++- plugins/grantforge-plugin-example/pom.xml | 5 ++ pom.xml | 95 +++++++++++++++++++++++ 3 files changed, 143 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f78df042..daf33cc5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,9 +5,11 @@ name: Release -# A release is a pushed tag v (D-83): script/ci/release.sh builds the distribution, its SBOM and checksums -# and takes the notes from the documentation site's changelog; this workflow then publishes the image to GHCR and -# the GitHub release. Versions with -rc.N are pre-releases and do not move the image's latest tag. +# A release is a pushed tag v, which script/release/tag.sh makes (D-83, D-90): script/ci/release.sh builds the +# distribution, its SBOM and checksums and writes the notes from the commits since the previous release; this +# workflow then publishes the image to GHCR, the Maven artifacts to GitHub Packages and, when the CENTRAL_USERNAME, +# CENTRAL_PASSWORD, GPG_PRIVATE_KEY and GPG_PASSPHRASE secrets are set, to Maven Central, and last the GitHub release. +# Versions with -rc.N are pre-releases and do not move the image's latest tag. on: push: tags: ['v*'] @@ -30,8 +32,13 @@ jobs: permissions: contents: write packages: write + env: + CENTRAL_CONFIGURED: ${{ secrets.CENTRAL_USERNAME != '' && secrets.GPG_PRIVATE_KEY != '' }} steps: - uses: actions/checkout@v6 + with: + # The notes list the commits since the previous release tag. + fetch-depth: 0 - uses: actions/setup-java@v6 with: distribution: temurin @@ -73,6 +80,39 @@ jobs: --label "org.opencontainers.image.licenses=MIT" \ --build-arg "RELEASE=grantforge-${VERSION}.tar.gz" \ -f deploy/docker/Dockerfile target/release + - uses: actions/setup-java@v6 + with: + distribution: temurin + java-version: '21' + server-id: github + server-username: GITHUB_ACTOR + server-password: GITHUB_TOKEN + - name: Publish the Maven artifacts to GitHub Packages + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + ./mvnw --batch-mode --no-transfer-progress -Prelease -Dgpg.skip -DskipTests deploy \ + "-DaltDeploymentRepository=github::https://maven.pkg.github.com/${GITHUB_REPOSITORY}" + - uses: actions/setup-java@v6 + if: env.CENTRAL_CONFIGURED == 'true' + with: + distribution: temurin + java-version: '21' + server-id: central + server-username: CENTRAL_USERNAME + server-password: CENTRAL_PASSWORD + gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Publish the Maven artifacts to Maven Central + if: env.CENTRAL_CONFIGURED == 'true' + env: + CENTRAL_USERNAME: ${{ secrets.CENTRAL_USERNAME }} + CENTRAL_PASSWORD: ${{ secrets.CENTRAL_PASSWORD }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + run: ./mvnw --batch-mode --no-transfer-progress -Prelease,central -Dcentral.skip=false -DskipTests deploy + - name: Skip Maven Central + if: env.CENTRAL_CONFIGURED != 'true' + run: echo "::notice::Maven Central skipped; set CENTRAL_USERNAME, CENTRAL_PASSWORD, GPG_PRIVATE_KEY and GPG_PASSPHRASE to publish there" - name: Publish the GitHub release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/plugins/grantforge-plugin-example/pom.xml b/plugins/grantforge-plugin-example/pom.xml index 8292c718..6806037d 100644 --- a/plugins/grantforge-plugin-example/pom.xml +++ b/plugins/grantforge-plugin-example/pom.xml @@ -25,6 +25,11 @@ the plugin host's tests and the full-stack tests install the packaged jar. Not part of the release. + + + true + + org.devlive.grantforge diff --git a/pom.xml b/pom.xml index f9b1e45d..c094ff5b 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,8 @@ https://github.com/devlive-community/grantforge + scm:git:https://github.com/devlive-community/grantforge.git + scm:git:git@github.com:devlive-community/grantforge.git @@ -108,6 +110,14 @@ 3.7.0 0.8.15 2.9.3 + + 3.3.1 + 3.12.0 + 3.2.8 + 0.11.0 + + true true 3.28.0 @@ -505,5 +515,90 @@ + + + + release + + + + org.apache.maven.plugins + maven-source-plugin + ${plugin.source.version} + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + ${plugin.javadoc.version} + + + none + true + + + + attach-javadocs + + jar + + + + + + org.apache.maven.plugins + maven-gpg-plugin + ${plugin.gpg.version} + + + sign-artifacts + verify + + sign + + + + --pinentry-mode + loopback + + + + + + + + + + + + central + + + + org.sonatype.central + central-publishing-maven-plugin + ${plugin.central.version} + true + + central + ${central.skip} + true + published + + grantforge-plugin-example + + + + + + From f7ab3895b9b3aeb9d9cb1f0bc00f524d4c66c087 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 07:36:28 -0400 Subject: [PATCH 19/22] build: skip the release bundles in the database tests The database jobs build a few modules with -pl, which leaves out the HDFS agent, so the server's prepare-package could not copy it and every database job failed. grantforge.bundle.skip turns the plugin and agent bundling off; the database tests set it, full builds keep bundling. --- core/grantforge-server/pom.xml | 2 ++ pom.xml | 2 ++ script/ci/db_integration.sh | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/core/grantforge-server/pom.xml b/core/grantforge-server/pom.xml index 16fab4a2..77066a8f 100644 --- a/core/grantforge-server/pom.xml +++ b/core/grantforge-server/pom.xml @@ -202,6 +202,7 @@ unpack + ${grantforge.bundle.skip} org.devlive.grantforge @@ -221,6 +222,7 @@ copy + ${grantforge.bundle.skip} org.devlive.grantforge diff --git a/pom.xml b/pom.xml index c094ff5b..5da9cca1 100644 --- a/pom.xml +++ b/pom.xml @@ -118,6 +118,8 @@ true + + false true 3.28.0 diff --git a/script/ci/db_integration.sh b/script/ci/db_integration.sh index b5188e55..61ff898b 100755 --- a/script/ci/db_integration.sh +++ b/script/ci/db_integration.sh @@ -17,5 +17,6 @@ fi ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "${ROOT}" -./mvnw --batch-mode --no-transfer-progress -DskipFrontend -Pdatabase-it \ +# The release's bundled plugins and agents are not built here (-pl), and the tests do not need them. +./mvnw --batch-mode --no-transfer-progress -DskipFrontend -Pdatabase-it -Dgrantforge.bundle.skip=true \ -pl core/grantforge-persistence,core/grantforge-audit,core/grantforge-identity,core/grantforge-authz,core/grantforge-server -am verify "-Dgrantforge.it.database=$1" From fd9dace57fb9b3e639d7c094bb4cb1b2ddde5205 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 07:36:28 -0400 Subject: [PATCH 20/22] ci: accept the licences of Hadoop's shaded protobuf The HDFS agent brings hadoop-shaded-protobuf_3_25, licensed under Apache-2.0, BSD-3-Clause, MIT and protobuf's BSD-style licence, all permissive; dependency review rejected the combination. --- .github/dependency-review-config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/dependency-review-config.yml b/.github/dependency-review-config.yml index 94211b79..5a6944c3 100644 --- a/.github/dependency-review-config.yml +++ b/.github/dependency-review-config.yml @@ -40,4 +40,6 @@ allow-dependencies-licenses: - pkg:npm/uri-js # The Bouncy Castle Licence is the MIT license under its own name, which the review does not recognise. - pkg:maven/org.bouncycastle/bcprov-jdk18on + # Hadoop's relocated protobuf for the HDFS agent: Apache-2.0, BSD-3-Clause, MIT and protobuf's BSD-style licence. + - pkg:maven/org.apache.hadoop.thirdparty/hadoop-shaded-protobuf_3_25 comment-summary-in-pr: on-failure From fcca96e61da3e7f3cb08e0cc591b202cc509780d Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 07:44:46 -0400 Subject: [PATCH 21/22] docs: rewrite the README in English with a Chinese edition The README describes the rebuilt platform in English, with the same content in README.zh-CN.md, and links the documentation at grantforge.devlive.org, where the site is now published. --- README.md | 153 ++++++++++++++++++++++++++++++++--------- README.zh-CN.md | 179 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 298 insertions(+), 34 deletions(-) create mode 100644 README.zh-CN.md diff --git a/README.md b/README.md index 967ec49e..4694e854 100644 --- a/README.md +++ b/README.md @@ -11,84 +11,169 @@ # GrantForge -开源权限管理平台 · 用户、角色、菜单与接口授权 +Unified permission platform · users, roles, menus, APIs, data rows and fields · external data systems + +Language: English · [中文说明](README.zh-CN.md) [![License](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) ![Version](https://img.shields.io/badge/version-2026.0.0-4F46E5) +![Java](https://img.shields.io/badge/Java-17%2B-ED8B00) +[![Docs](https://img.shields.io/badge/docs-grantforge.devlive.org-4F46E5)](https://grantforge.devlive.org) +[![Docker](https://img.shields.io/badge/ghcr.io-grantforge-2496ED)](https://ghcr.io/devlive-community/grantforge) -GrantForge（原 AuthX）是开源（MIT）的插件化细粒度权限平台，目标覆盖页面、按钮、API、数据行与字段级授权，并可通过插件接管外部系统（如 HDFS、Hive）的权限。当前版本号为 `2026.0.0`。 +GrantForge (formerly AuthX) is an open-source (MIT) unified permission platform. It answers two questions in one place: **who can do what** (functional authorization) and **who can see which data** (data and field authorization). Permissions are defined, explained and audited in the console, applications integrate through standard protocols, and external data systems such as HDFS are brought into the same policy model through plug-ins and agents. -> `rebuild` 分支正在基于 Spring Boot 4.1 / Java 17 整体重建。重建期间旧版后端保留在 `dev` 分支与 `legacy-2026.0.0` 标签中；在新接口完成前，Web 管理界面的数据功能不可用。 +

+ GrantForge console +

-## 项目结构 +## Features -| 模块 | 职责 | +| Area | What it does | | --- | --- | -| `core/grantforge-server` | Spring Boot 服务入口，提供 REST API 并托管 Web 管理界面 | -| `core/grantforge-web` | Vue 3 / TypeScript / Tailwind CSS 管理界面 | -| `script/ci` | CI 检查脚本（许可证头、格式、测试映射、安全扫描等），本地与 CI 使用同一脚本 | +| Identity & organization | Multi-tenancy, department tree, user groups and positions; CSV bulk import/export; LDAP / Active Directory sign-in and sync, OIDC federation | +| Account security | Session management and forced sign-out, password policy with lockout, TOTP two-factor authentication with recovery codes, step-up verification for sensitive operations | +| Functional authorization | Resource catalog (modules, menus, pages, tabs, buttons, APIs), role inheritance, grant matrix, impact analysis before granting | +| Data permissions | Row-level conditions (self, own department and descendants, specified departments, custom conditions), with read and write controlled separately | +| Field permissions | Fields can be hidden, masked (email, phone number, ID number) or read-only | +| Explainability & audit | Permission explanation (where every grant comes from), grant simulation, audit log query and export | +| Governance | Separation of duty constraints, access requests with approval, periodic access reviews | +| Application integration | OAuth 2.1 / OIDC authorization server, permission open API, Java (Spring Boot starter) and JavaScript SDK | +| External systems | Plug-in service types and a policy engine: data services, access policies, agents and access auditing | +| Delivery | One executable release, Docker image, Compose examples, Helm chart; H2, PostgreSQL, MySQL, MariaDB, Oracle, SQL Server | + +External-system support today ships the plug-in framework, the generic policy editor, signed policy distribution, access auditing, an HDFS service type and a Hadoop 3.5.0 NameNode agent; the Hive plug-in and agents for other Hadoop versions are still in progress. + +## How it works: two planes + +- **Management plane**: the GrantForge server (Spring Boot 4.1, Java 17 bytecode) and the Vue 3 console own tenants, accounts, organization, roles, grants, auditing, data services and policies. +- **Data plane**: agents embedded in the system being protected. An agent pulls Ed25519-signed policy snapshots with its token and caches them locally, decides every access before it happens (denying when no policy is available), and reports access events back for auditing. + +Your own systems do not have to follow the HDFS pattern. Regular applications evaluate permissions in-process through the open API or the Spring Boot starter; only systems that must intercept access inside a database, file system or similar store need an agent written against `core/grantforge-agent-core`. + +## Integrating your application + +- **OAuth 2.1 / OpenID Connect**: GrantForge is an authorization server, so applications sign users in with it; existing identity sources (LDAP / AD / OIDC) can also be connected. +- **Java applications**: `sdk/grantforge-spring-boot-starter` adds `@RequirePermission` for endpoints, `@GrantForgeEntity` for data entities, and `GrantForgeDataScopes.scope(...)` to turn platform data permissions into JPA `Specification`s. +- **Front-end applications**: `@grantforge/client` signs users in with OIDC + PKCE from your own origin and queries their permissions. +- **Open API**: `/api/v1/open/me/authorization`, `/api/v1/open/me/data-access`, `/api/v1/open/catalog/data-entities`. +- **Runnable examples**: `samples/shop` and `samples/notes` integrate the way a third party would. + +## Quick start + +Java 17 or later is required. The service listens on port `9999` and prints a one-time **setup token** on first start; open in a browser, enter the token and create the first administrator. -Maven 根坐标：`org.devlive.grantforge:grantforge:2026.0.0`。Java 包前缀：`org.devlive.grantforge`。启动类：`org.devlive.grantforge.server.GrantForge`。 +```bash +# From the release (or build it from source with ./mvnw clean package, output in dist/) +tar -xzf grantforge-release.tar.gz +cd grantforge +bin/startup.sh -## 数据库 +# Or with Docker +docker run -p 9999:9999 ghcr.io/devlive-community/grantforge:2026.0.0 -默认使用内嵌 H2 文件库（`${GRANTFORGE_HOME}/data`），无需任何配置即可启动。生产环境通过环境变量切换，库表结构由 Liquibase 统一管理： +# Or with Compose against a database +docker compose -f deploy/compose/postgres.yml up -d -| 数据库 | 版本（CI 验证） | `GRANTFORGE_DB_URL` 示例 | +# Or on Kubernetes +helm install grantforge deploy/helm/grantforge \ + --set database.url=jdbc:postgresql://postgresql:5432/grantforge \ + --set database.username=grantforge \ + --set encryptionKey=$(openssl rand -base64 32) +``` + +The embedded H2 file database is the default, so no configuration is needed to start. The MySQL driver is not distributed with the release because of its GPL license; put it into `drivers/`. Installation, first-run setup and your first grant are described in the [documentation](https://grantforge.devlive.org). + +## Databases + +The default is an embedded H2 file database (`${GRANTFORGE_HOME}/data`), so no configuration is needed to start. Production deployments switch through environment variables; the schema is managed by Liquibase: + +| Database | Versions (verified in CI) | `GRANTFORGE_DB_URL` example | | --- | --- | --- | -| PostgreSQL | 14、17 | `jdbc:postgresql://host:5432/grantforge` | -| MySQL | 8.0、8.4 | `jdbc:mysql://host:3306/grantforge`（需自行将 `mysql-connector-j` 放入 `lib/`，其 GPL 许可不随发行包分发） | -| MariaDB | 10.11、11.4 | `jdbc:mariadb://host:3306/grantforge` | +| PostgreSQL | 14, 17 | `jdbc:postgresql://host:5432/grantforge` | +| MySQL | 8.0, 8.4 | `jdbc:mysql://host:3306/grantforge` (add `mysql-connector-j` to `lib/`; its GPL license keeps it out of the release) | +| MariaDB | 10.11, 11.4 | `jdbc:mariadb://host:3306/grantforge` | | Oracle | 23 | `jdbc:oracle:thin:@//host:1521/FREEPDB1` | | SQL Server | 2022 | `jdbc:sqlserver://host:1433;databaseName=grantforge;encrypt=true` | -同时设置 `GRANTFORGE_DB_USER`、`GRANTFORGE_DB_PASSWORD`；集群部署时每个实例必须设置不同的 `GRANTFORGE_ID_NODE`（0-1023）。 - -## 运维与可观测性 +Also set `GRANTFORGE_DB_USER` and `GRANTFORGE_DB_PASSWORD`; every instance in a cluster must set its own `GRANTFORGE_ID_NODE` (0-1023). -- 健康探针：`/actuator/health/liveness`、`/actuator/health/readiness`（仅返回状态，不暴露细节）。 -- 指标：`/actuator/prometheus`（带 `application="grantforge"` 标签）。 -- 日志：默认可读文本，每行带请求编号；设置 `LOGGING_STRUCTURED_FORMAT_CONSOLE=ecs`（或 `logstash`）输出 JSON 日志。 +## Project layout -## 开发与验证 +Maven root coordinate: `org.devlive.grantforge:grantforge:2026.0.0`. Java package prefix: `org.devlive.grantforge`. Main class: `org.devlive.grantforge.server.GrantForge`. -构建需要 JDK 17 或更高版本（产物目标为 Java 17）；在 JDK 21+ 上会自动启用 Error Prone + NullAway 空值检查。前端使用 Vue 3.5、Tailwind CSS 4、Node.js 22.12+ 和 pnpm 8.10.2。 +| Module | Responsibility | +| --- | --- | +| `core/grantforge-server` | Spring Boot entry point: REST API, security configuration, open API, and it serves the web console | +| `core/grantforge-web` | Vue 3 / TypeScript / Tailwind CSS console | +| `core/grantforge-common` | Error codes and problem details, CSV, endpoint access annotations | +| `core/grantforge-persistence` | Entities, tenant filtering, TSID, Liquibase, data and field permission SPI | +| `core/grantforge-audit` | Audit event recording, querying, retention and archiving | +| `core/grantforge-identity` | Tenants, accounts, departments, groups, positions, sign-in and sessions, two-factor authentication, identity sources | +| `core/grantforge-authz` | Resource catalog, roles, grants, assignments and evaluation, data and field policies, separation of duty, requests and reviews | +| `core/grantforge-plugin-api` / `core/grantforge-plugin-host` | Service type plug-in contract plus loading, isolation and invocation of plug-ins | +| `core/grantforge-policy-engine` | Policy evaluation engine for external systems (Java 8 API, embeddable in agents) | +| `core/grantforge-agent-core` | Shared agent code: settings, signed snapshots, access decisions, audit shipping | +| `core/grantforge-service` | Data services, policy snapshot signing and distribution, agents and access auditing | +| `core/grantforge-oauth` | OAuth 2.1 / OIDC server built on Spring Authorization Server | +| `plugins/grantforge-plugin-hdfs` | HDFS service type plug-in: policy management and resource lookup | +| `plugins/grantforge-agent-hdfs` | Hadoop 3.5.0 NameNode agent: overlay authorization and access auditing | +| `plugins/grantforge-plugin-example` | Example plug-in for a custom service type | +| `sdk/grantforge-spring-boot-starter`, `sdk/grantforge-js` | Java and JavaScript SDKs for integrating applications | +| `script/ci`, `deploy/` | CI check scripts (the same ones locally and in CI) and deployment resources (Dockerfile, Compose, Helm) | + +## Operations and observability + +- Health probes: `/actuator/health/liveness`, `/actuator/health/readiness` (status only, no details; readiness returns 200 once the database is reachable and migrations have run). +- Metrics: `/actuator/prometheus` (labelled `application="grantforge"`, login required by default; open it to trusted networks with `GRANTFORGE_PROMETHEUS_PUBLIC=true`). +- Logging: readable text with a request ID per line by default; set `LOGGING_STRUCTURED_FORMAT_CONSOLE=ecs` (or `logstash`) for JSON logs. +- Release scripts: `bin/startup.sh`, `shutdown.sh`, `restart.sh`, `debug.sh` and `import-legacy.sh`. + +## Development and verification + +Builds need JDK 17 or later (Java 17 bytecode; the policy engine targets Java 8); Error Prone + NullAway enable themselves on JDK 21+. The front end uses Vue 3.5, Tailwind CSS 4, Node.js 22.12+ and pnpm 8.10.2. ```sh -# Java 构建与单元测试（跳过前端构建） +# Java build and unit tests (skipping the console build) +./mvnw verify bash script/ci/java.sh test bash script/ci/java.sh checkstyle -# 在指定数据库上运行持久化集成测试（需要 Docker，h2 除外） +# Persistence integration tests on a given database (needs Docker, except h2) bash script/ci/db_integration.sh postgres:17 -# 打包发布包（包含前端构建），输出到 dist/ +# Package the release (including the console build) into dist/ ./mvnw clean package -# 前端开发与检查 +# Front-end development and checks bash script/ci/web.sh install cd core/grantforge-web && pnpm dev bash script/ci/web.sh lint -# API 契约：服务端接口变更后重新生成 openapi.json 与前端类型（CI 会校验两者一致） +# Sample applications and SDKs +cd sdk/grantforge-js && pnpm install && pnpm build +./mvnw -f samples/pom.xml package -DskipTests + +# API contract: regenerate openapi.json and the front-end types after a server change (CI checks both) ./mvnw -DskipFrontend -pl core/grantforge-server -am test -Dtest=OpenApiContractTest \ -Dsurefire.failIfNoSpecifiedTests=false -Dgrantforge.openapi.update=true cd core/grantforge-web && pnpm api:generate -# 文档站（docs/，Next.js + Tailwind CSS） +# Documentation site (docs/, Next.js + Tailwind CSS) bash script/ci/docs.sh install cd docs && pnpm dev # http://localhost:3100 bash script/ci/docs.sh check -bash script/docs/screenshots.sh # 用真实服务与示例数据重新生成文档截图 +bash script/docs/screenshots.sh # regenerate the screenshots with a real service and sample data -# 仓库检查（与 CI 相同） +# Repository checks (the same ones CI runs) python3 script/ci/check_license_headers.py bash script/ci/test_ci_scripts.sh ``` -## 项目链接 +## Links -- [项目仓库](https://github.com/devlive-community/grantforge) -- [文档站](https://authx.devlive.org)：快速开始、使用指南、应用接入与技术文档，源文件在 [`docs/`](docs/) +- [Repository](https://github.com/devlive-community/grantforge) +- [Documentation](https://grantforge.devlive.org): quick start, user guide, integration and technical references, sources in [`docs/`](docs/) +- [Contributing](CONTRIBUTING.md) · [Code of conduct](CODE_OF_CONDUCT.md) · [Changelog](CHANGELOG) diff --git a/README.zh-CN.md b/README.zh-CN.md new file mode 100644 index 00000000..788c87bd --- /dev/null +++ b/README.zh-CN.md @@ -0,0 +1,179 @@ + + +
+ +GrantForge logo + +# GrantForge + +统一权限平台 · 用户、角色、菜单、接口、数据行与字段授权 · 外部数据系统 + +语言：[English](README.md) · 中文 + +[![License](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) +![Version](https://img.shields.io/badge/version-2026.0.0-4F46E5) +![Java](https://img.shields.io/badge/Java-17%2B-ED8B00) +[![Docs](https://img.shields.io/badge/docs-grantforge.devlive.org-4F46E5)](https://grantforge.devlive.org) +[![Docker](https://img.shields.io/badge/ghcr.io-grantforge-2496ED)](https://ghcr.io/devlive-community/grantforge) + +
+ +GrantForge（原 AuthX）是一个开源（MIT）的统一权限平台。它集中回答两个问题：**谁能做什么**（功能授权）和**谁能看到哪些数据**（数据与字段授权）。权限在控制台里定义、解释与审计，业务应用通过标准协议接入，外部数据系统（例如 HDFS）则通过插件与代理纳入同一套策略体系。 + +

+ GrantForge 控制台 +

+ +## 功能 + +| 领域 | 功能 | +| --- | --- | +| 身份与组织 | 多租户、部门树、用户组与岗位；CSV 批量导入导出；LDAP / Active Directory 登录与同步、OIDC 联合登录 | +| 账号安全 | 会话管理与强制下线、密码策略与失败锁定、TOTP 两步验证与恢复码、敏感操作二次验证 | +| 功能授权 | 资源目录（模块、菜单、页面、标签、按钮、API）、角色继承、授权矩阵、授权影响分析 | +| 数据权限 | 按条件限定可见的数据行（本人、本部门及下级、指定部门、自定义条件），读与写分开控制 | +| 字段权限 | 字段可隐藏、可脱敏（邮箱、手机号、证件号等）、可只读 | +| 可解释与审计 | 权限解释（每项权限从哪来）、授权模拟、审计日志查询与导出 | +| 治理 | 职责分离（SOD）约束、权限申请与审批、定期权限复核 | +| 应用接入 | OAuth 2.1 / OIDC 授权服务器、权限查询开放 API、Java（Spring Boot Starter）与 JavaScript SDK | +| 外部系统 | 插件化服务类型与策略引擎：数据服务、访问策略、代理与访问审计 | +| 交付 | 单一可执行发行包、Docker 镜像、Compose 示例、Helm Chart；H2、PostgreSQL、MySQL、MariaDB、Oracle、SQL Server | + +外部系统支持当前提供插件框架、通用策略编辑器、策略签名分发、访问审计，以及 HDFS 服务类型插件与 Hadoop 3.5.0 NameNode 代理；Hive 插件与其他 Hadoop 版本的代理仍在开发中。 + +## 工作原理：两个平面 + +- **管理平面**：GrantForge 服务端（Spring Boot 4.1，Java 17 字节码）与 Vue 3 控制台，负责租户、账号、组织、角色、授权、审计，以及数据服务与策略的维护。 +- **数据平面**：嵌入受保护系统的代理。代理凭令牌定期从服务端拉取带 Ed25519 签名的策略快照并缓存在本地，在每次访问之前判定允许或拒绝（策略不可达时默认拒绝），同时把访问事件回传服务端审计。 + +自己的系统不必照抄 HDFS 的做法：普通业务应用用开放 API 或 Spring Boot Starter 在进程内求值即可，只有需要在数据库、文件系统等存储系统内部拦截访问时，才需要用 `core/grantforge-agent-core` 编写嵌入目标系统的代理。 + +## 接入你的应用 + +- **OAuth 2.1 / OpenID Connect**：GrantForge 本身就是授权服务器，应用用它登录用户；已有的身份源（LDAP / AD / OIDC）也可以接进来。 +- **Java 应用**：`sdk/grantforge-spring-boot-starter` 提供 `@RequirePermission` 接口鉴权、`@GrantForgeEntity` 声明数据实体，以及 `GrantForgeDataScopes.scope(...)` 把平台上的数据权限转成 JPA `Specification`。 +- **前端应用**：`@grantforge/client` 用 OIDC + PKCE 从你自己的域名引导用户登录，并查询当前用户的权限。 +- **开放 API**：`/api/v1/open/me/authorization`、`/api/v1/open/me/data-access`、`/api/v1/open/catalog/data-entities`。 +- **可运行的例子**：`samples/` 下的 `shop` 与 `notes` 两个应用按第三方的方式接入。 + +## 快速开始 + +需要 Java 17 或更高版本。服务默认监听 `9999` 端口，首次启动会在日志里打印一次性的**初始化令牌**；用浏览器打开 ，填入令牌并创建第一个管理员即可。 + +```bash +# 使用发行包（或从源码 ./mvnw clean package 构建，产物在 dist/） +tar -xzf grantforge-release.tar.gz +cd grantforge +bin/startup.sh + +# 或使用 Docker +docker run -p 9999:9999 ghcr.io/devlive-community/grantforge:2026.0.0 + +# 或用 Compose 搭配数据库 +docker compose -f deploy/compose/postgres.yml up -d + +# 或部署到 Kubernetes +helm install grantforge deploy/helm/grantforge \ + --set database.url=jdbc:postgresql://postgresql:5432/grantforge \ + --set database.username=grantforge \ + --set encryptionKey=$(openssl rand -base64 32) +``` + +默认使用内嵌 H2 文件库，无需任何配置即可启动。MySQL 驱动因 GPL 许可不随发行包分发，需要自行放进 `drivers/`。安装、初始化与第一次授权的详细步骤见[文档站](https://grantforge.devlive.org)。 + +## 数据库 + +默认使用内嵌 H2 文件库（`${GRANTFORGE_HOME}/data`），无需任何配置即可启动。生产环境通过环境变量切换，库表结构由 Liquibase 统一管理： + +| 数据库 | 版本（CI 验证） | `GRANTFORGE_DB_URL` 示例 | +| --- | --- | --- | +| PostgreSQL | 14、17 | `jdbc:postgresql://host:5432/grantforge` | +| MySQL | 8.0、8.4 | `jdbc:mysql://host:3306/grantforge`（需自行将 `mysql-connector-j` 放入 `lib/`，其 GPL 许可不随发行包分发） | +| MariaDB | 10.11、11.4 | `jdbc:mariadb://host:3306/grantforge` | +| Oracle | 23 | `jdbc:oracle:thin:@//host:1521/FREEPDB1` | +| SQL Server | 2022 | `jdbc:sqlserver://host:1433;databaseName=grantforge;encrypt=true` | + +同时设置 `GRANTFORGE_DB_USER`、`GRANTFORGE_DB_PASSWORD`；集群部署时每个实例必须设置不同的 `GRANTFORGE_ID_NODE`（0-1023）。 + +## 项目结构 + +Maven 根坐标：`org.devlive.grantforge:grantforge:2026.0.0`。Java 包前缀：`org.devlive.grantforge`。启动类：`org.devlive.grantforge.server.GrantForge`。 + +| 模块 | 职责 | +| --- | --- | +| `core/grantforge-server` | Spring Boot 服务入口：REST API、安全配置、开放 API，并托管 Web 管理界面 | +| `core/grantforge-web` | Vue 3 / TypeScript / Tailwind CSS 管理界面 | +| `core/grantforge-common` | 错误码与 problem details 模型、CSV、接口访问注解 | +| `core/grantforge-persistence` | 实体、租户过滤、TSID、Liquibase、数据与字段权限 SPI | +| `core/grantforge-audit` | 审计事件的记录、查询、保留与归档 | +| `core/grantforge-identity` | 租户、账号、部门、用户组、岗位、登录与会话、两步验证、身份源 | +| `core/grantforge-authz` | 资源目录、角色、授权、分配与求值、数据与字段策略、职责分离、申请与复核 | +| `core/grantforge-plugin-api` / `core/grantforge-plugin-host` | 服务类型插件的契约，以及插件的加载、隔离与调用 | +| `core/grantforge-policy-engine` | 外部系统策略的求值引擎（Java 8 API，可嵌入代理） | +| `core/grantforge-agent-core` | 代理的公共代码：设置、签名快照、访问判定、审计上报 | +| `core/grantforge-service` | 数据服务、策略快照签名与分发、代理与访问审计 | +| `core/grantforge-oauth` | 基于 Spring Authorization Server 的 OAuth 2.1 / OIDC 服务器 | +| `plugins/grantforge-plugin-hdfs` | HDFS 服务类型插件：策略管理与资源查询 | +| `plugins/grantforge-agent-hdfs` | Hadoop 3.5.0 NameNode 代理：覆盖式授权与访问审计 | +| `plugins/grantforge-plugin-example` | 自定义服务类型的示例插件 | +| `sdk/grantforge-spring-boot-starter`、`sdk/grantforge-js` | Java 与 JavaScript 应用接入 SDK | +| `script/ci`、`deploy/` | CI 检查脚本（本地与 CI 使用同一脚本）与部署资源（Dockerfile、Compose、Helm） | + +## 运维与可观测性 + +- 健康探针：`/actuator/health/liveness`、`/actuator/health/readiness`（仅返回状态，不暴露细节；就绪探针在数据库可用且迁移完成后才返回 200）。 +- 指标：`/actuator/prometheus`（带 `application="grantforge"` 标签，默认需要登录，可用 `GRANTFORGE_PROMETHEUS_PUBLIC=true` 对受信网络开放）。 +- 日志：默认可读文本，每行带请求编号；设置 `LOGGING_STRUCTURED_FORMAT_CONSOLE=ecs`（或 `logstash`）输出 JSON 日志。 +- 发行包脚本：`bin/` 下的 `startup.sh`、`shutdown.sh`、`restart.sh`、`debug.sh` 与 `import-legacy.sh`。 + +## 开发与验证 + +构建需要 JDK 17 或更高版本（产物目标为 Java 17 字节码，策略引擎为 Java 8）；在 JDK 21+ 上会自动启用 Error Prone + NullAway 空值检查。前端使用 Vue 3.5、Tailwind CSS 4、Node.js 22.12+ 和 pnpm 8.10.2。 + +```sh +# Java 构建与单元测试（跳过前端构建） +./mvnw verify +bash script/ci/java.sh test +bash script/ci/java.sh checkstyle + +# 在指定数据库上运行持久化集成测试（需要 Docker，h2 除外） +bash script/ci/db_integration.sh postgres:17 + +# 打包发布包（包含前端构建），输出到 dist/ +./mvnw clean package + +# 前端开发与检查 +bash script/ci/web.sh install +cd core/grantforge-web && pnpm dev +bash script/ci/web.sh lint + +# 示例应用与 SDK +cd sdk/grantforge-js && pnpm install && pnpm build +./mvnw -f samples/pom.xml package -DskipTests + +# API 契约：服务端接口变更后重新生成 openapi.json 与前端类型（CI 会校验两者一致） +./mvnw -DskipFrontend -pl core/grantforge-server -am test -Dtest=OpenApiContractTest \ + -Dsurefire.failIfNoSpecifiedTests=false -Dgrantforge.openapi.update=true +cd core/grantforge-web && pnpm api:generate + +# 文档站（docs/，Next.js + Tailwind CSS） +bash script/ci/docs.sh install +cd docs && pnpm dev # http://localhost:3100 +bash script/ci/docs.sh check +bash script/docs/screenshots.sh # 用真实服务与示例数据重新生成文档截图 + +# 仓库检查（与 CI 相同） +python3 script/ci/check_license_headers.py +bash script/ci/test_ci_scripts.sh +``` + +## 项目链接 + +- [项目仓库](https://github.com/devlive-community/grantforge) +- [文档站](https://grantforge.devlive.org)：快速开始、使用指南、应用接入与技术文档，源文件在 [`docs/`](docs/) +- [贡献指南](CONTRIBUTING.md) · [行为准则](CODE_OF_CONDUCT.md) · [更新日志](CHANGELOG) From 548b03eed31dd4f62656fb1c7c0e1a672959895b Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Mon, 5 Oct 2026 07:48:00 -0400 Subject: [PATCH 22/22] ci: accept the licences of the Javadoc build plugin maven-javadoc-plugin, which writes the release's Javadoc jars and is never shipped, combines Apache-2.0 with BSD, MIT and public domain parts, which dependency review rejected. --- .github/dependency-review-config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/dependency-review-config.yml b/.github/dependency-review-config.yml index 5a6944c3..be7d1488 100644 --- a/.github/dependency-review-config.yml +++ b/.github/dependency-review-config.yml @@ -42,4 +42,6 @@ allow-dependencies-licenses: - pkg:maven/org.bouncycastle/bcprov-jdk18on # Hadoop's relocated protobuf for the HDFS agent: Apache-2.0, BSD-3-Clause, MIT and protobuf's BSD-style licence. - pkg:maven/org.apache.hadoop.thirdparty/hadoop-shaded-protobuf_3_25 + # A build plugin that writes the release's Javadoc jars, never shipped: Apache-2.0 with BSD, MIT and public domain parts. + - pkg:maven/org.apache.maven.plugins/maven-javadoc-plugin comment-summary-in-pr: on-failure