From 080c52590a8baf02c06ba1d0cb8ac16d76e1c3dc Mon Sep 17 00:00:00 2001 From: Dmitry Kasimovskiy Date: Wed, 3 Jun 2026 17:01:32 +0300 Subject: [PATCH 1/5] feat(testcontainers): add TQE 2.x support with versioned cluster implementations Add TQE 2.x (message-queue-ee 2.x) integration with separate cluster, configurator, and gRPC layer. Refactor TQE 3.x into versioned structure alongside TQE 2.x, extract AbstractTQECluster with Template Method pattern, migrate from xolstice to ascopes protobuf-maven-plugin. Co-Authored-By: Claude Opus 4.7 --- testcontainers/pom.xml | 51 ++- ...usterImpl.java => AbstractTQECluster.java} | 97 ++-- .../containers/tqe/GrpcContainer.java | 4 +- .../containers/tqe/GrpcContainerImpl.java | 6 +- .../containers/tqe/TQE2ClusterImpl.java | 19 + .../containers/tqe/TQE3ClusterImpl.java | 33 ++ .../configuration/FileTQEConfigurator.java | 179 +++++--- .../configuration/grpc/GrpcConfiguration.java | 2 + .../integration/tqe/AbstractTQETest.java | 161 +++++++ .../integration/tqe/CommonTest.java | 48 -- ...est.java => FileTQE2ConfiguratorTest.java} | 63 ++- .../tqe/FileTQE3ConfiguratorTest.java | 267 +++++++++++ .../integration/tqe/TQE2ClusterImplTest.java | 429 ++++++++++++++++++ ...ImplTest.java => TQE3ClusterImplTest.java} | 338 +++++++------- .../test/proto/tqe2/messages/message.proto | 53 +++ .../test/proto/tqe2/services/consumer.proto | 58 +++ .../test/proto/tqe2/services/publisher.proto | 263 +++++++++++ .../proto/{ => tqe3}/messages/cursor.proto | 0 .../proto/{ => tqe3}/messages/message.proto | 0 .../proto/{ => tqe3}/services/consumer.proto | 0 .../proto/{ => tqe3}/services/producer.proto | 0 .../tqe2/simple-config/simple-grpc.yml | 21 + .../tqe2/simple-config/simple-queue.yml | 66 +++ .../simple-config/simple-grpc.yml | 6 +- .../simple-config/simple-queue.yml | 0 25 files changed, 1798 insertions(+), 366 deletions(-) rename testcontainers/src/main/java/org/testcontainers/containers/tqe/{TQEClusterImpl.java => AbstractTQECluster.java} (51%) create mode 100644 testcontainers/src/main/java/org/testcontainers/containers/tqe/TQE2ClusterImpl.java create mode 100644 testcontainers/src/main/java/org/testcontainers/containers/tqe/TQE3ClusterImpl.java create mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQETest.java delete mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/CommonTest.java rename testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/{FileTQEConfiguratorTest.java => FileTQE2ConfiguratorTest.java} (76%) create mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE3ConfiguratorTest.java create mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java rename testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/{TQEClusterImplTest.java => TQE3ClusterImplTest.java} (57%) create mode 100644 testcontainers/src/test/proto/tqe2/messages/message.proto create mode 100644 testcontainers/src/test/proto/tqe2/services/consumer.proto create mode 100644 testcontainers/src/test/proto/tqe2/services/publisher.proto rename testcontainers/src/test/proto/{ => tqe3}/messages/cursor.proto (100%) rename testcontainers/src/test/proto/{ => tqe3}/messages/message.proto (100%) rename testcontainers/src/test/proto/{ => tqe3}/services/consumer.proto (100%) rename testcontainers/src/test/proto/{ => tqe3}/services/producer.proto (100%) create mode 100644 testcontainers/src/test/resources/tqe2/simple-config/simple-grpc.yml create mode 100644 testcontainers/src/test/resources/tqe2/simple-config/simple-queue.yml rename testcontainers/src/test/resources/{tqe => tqe3}/simple-config/simple-grpc.yml (84%) rename testcontainers/src/test/resources/{tqe => tqe3}/simple-config/simple-queue.yml (100%) diff --git a/testcontainers/pom.xml b/testcontainers/pom.xml index 9669f10c..ef86938b 100644 --- a/testcontainers/pom.xml +++ b/testcontainers/pom.xml @@ -18,7 +18,7 @@ ${project.parent.basedir}/LICENSE_HEADER.txt 17 - 0.6.1 + 5.1.4 @@ -104,13 +104,6 @@ - - - kr.motd.maven - os-maven-plugin - 1.7.0 - - org.jsonschema2pojo @@ -146,25 +139,45 @@ - org.xolstice.maven.plugins + io.github.ascopes protobuf-maven-plugin ${protobuf-plugin.version} - ${project.basedir}/src/test/proto - - com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} - - grpc-java - - io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} - + + ${protobuf.version} + + + + io.grpc + protoc-gen-grpc-java + ${grpc.version} + + + generate-test-tqe3 - test-compile - test-compile-custom + generate-test + + + ${project.basedir}/src/test/proto/tqe3 + + ${project.build.directory}/generated-test-sources/protobuf/tqe3 + + + + generate-test-tqe2 + + generate-test + + + + ${project.basedir}/src/test/proto/tqe2 + + ${project.build.directory}/generated-test-sources/protobuf/tqe2 + diff --git a/testcontainers/src/main/java/org/testcontainers/containers/tqe/TQEClusterImpl.java b/testcontainers/src/main/java/org/testcontainers/containers/tqe/AbstractTQECluster.java similarity index 51% rename from testcontainers/src/main/java/org/testcontainers/containers/tqe/TQEClusterImpl.java rename to testcontainers/src/main/java/org/testcontainers/containers/tqe/AbstractTQECluster.java index bcc85aa9..5015eaaf 100644 --- a/testcontainers/src/main/java/org/testcontainers/containers/tqe/TQEClusterImpl.java +++ b/testcontainers/src/main/java/org/testcontainers/containers/tqe/AbstractTQECluster.java @@ -10,6 +10,8 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,20 +20,32 @@ import org.testcontainers.containers.tqe.configuration.TQEConfigurator; import org.testcontainers.lifecycle.Startable; -public class TQEClusterImpl implements TQECluster { +/** + * Abstract base class for TQE cluster implementations that provides common functionality for + * different TQE versions (2.x, 3.x, etc.). + */ +public abstract class AbstractTQECluster implements TQECluster { + + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractTQECluster.class); - private static final Logger LOGGER = LoggerFactory.getLogger(TQEClusterImpl.class); + private static final ExecutorService STARTUP_EXECUTOR = + Executors.newCachedThreadPool( + r -> { + var t = new Thread(r, "tqe-cluster-startup"); + t.setDaemon(true); + return t; + }); - private final TQEConfigurator configurator; + protected final TQEConfigurator configurator; - private boolean isClosed; + private volatile boolean isClosed; - public TQEClusterImpl(TQEConfigurator configurator) { + public AbstractTQECluster(TQEConfigurator configurator) { this.configurator = configurator; } @Override - public synchronized void start() { + public final synchronized void start() { if (this.isClosed) { throw new IllegalStateException("Container is already closed. Please create new container"); } @@ -47,18 +61,43 @@ public synchronized void start() { @Override public synchronized void stop() { - if (this.configurator != null) { - try { - this.configurator.close(); - this.isClosed = true; - } catch (Exception e) { - throw new RuntimeException(e); - } + try { + this.configurator.close(); + this.isClosed = true; + } catch (Exception e) { + throw new RuntimeException(e); } } - private static void startParallel( + /** + * Starts Tarantool cluster nodes and configures the cluster. Default implementation starts all + * queue containers in parallel and then configures the cluster. Subclasses can override to change + * startup order. + */ + protected void startTarantoolCluster() { + startParallel(this.configurator.queue(), this.configurator); + if (!this.configurator.isConfigured()) { + this.configurator.configure(); + } + } + + /** + * Starts gRPC endpoint containers. Default implementation starts all gRPC containers in parallel. + * Subclasses can override to change startup order. + */ + protected void startGrpcEndpoints() { + startParallel(this.configurator.grpc(), this.configurator); + } + + /** + * Common utility method to start containers in parallel. + * + * @param containers map of container names to container instances + * @param configurator the TQE configurator + */ + protected static void startParallel( Map containers, TQEConfigurator configurator) { + final List> futures = new ArrayList<>(containers.size()); final CopyOnWriteArrayList errors = new CopyOnWriteArrayList<>(); @@ -70,12 +109,18 @@ private static void startParallel( try { c.start(); } catch (Exception e) { - LOGGER.error("Error starting TQE container [container_name='{}']", name, e); + LOGGER.error( + "Error starting TQE container [cluster='{}', container_name='{}']: {}", + configurator.clusterName(), + name, + e.getMessage(), + e); errors.add(e); } - }))); + }, + STARTUP_EXECUTOR))); - CompletableFuture.allOf(futures.toArray(new CompletableFuture[] {})).join(); + CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new)).join(); if (!errors.isEmpty()) { throw new ContainerLaunchException( @@ -85,24 +130,6 @@ private static void startParallel( } } - private void startTarantoolCluster() { - startParallel(this.configurator.queue(), this.configurator); - if (this.configurator.isConfigured()) { - LOGGER.warn( - "TQE cluster [name = {}, queue = {}, grpc = {}] already configured", - this.configurator.clusterName(), - this.configurator.queue().size(), - this.configurator.grpc().size()); - return; - } - - this.configurator.configure(); - } - - private void startGrpcEndpoints() { - startParallel(this.configurator.grpc(), this.configurator); - } - @Override public String clusterName() { return this.configurator.clusterName(); diff --git a/testcontainers/src/main/java/org/testcontainers/containers/tqe/GrpcContainer.java b/testcontainers/src/main/java/org/testcontainers/containers/tqe/GrpcContainer.java index 74883e51..ba66b3bd 100644 --- a/testcontainers/src/main/java/org/testcontainers/containers/tqe/GrpcContainer.java +++ b/testcontainers/src/main/java/org/testcontainers/containers/tqe/GrpcContainer.java @@ -50,7 +50,9 @@ public interface GrpcContainer> enum GrpcRole { CONSUMER("consumer"), - PRODUCER("producer"); + PRODUCER("producer"), + + PUBLISHER("publisher"); private final String role; diff --git a/testcontainers/src/main/java/org/testcontainers/containers/tqe/GrpcContainerImpl.java b/testcontainers/src/main/java/org/testcontainers/containers/tqe/GrpcContainerImpl.java index aba179a4..ccd15aba 100644 --- a/testcontainers/src/main/java/org/testcontainers/containers/tqe/GrpcContainerImpl.java +++ b/testcontainers/src/main/java/org/testcontainers/containers/tqe/GrpcContainerImpl.java @@ -227,7 +227,11 @@ private static Set resolveRoles(GrpcConfiguration config, Path configP final Set roles = new LinkedHashSet<>(); if (isPublisher.isPresent() && isPublisher.get()) { roles.add(GrpcRole.PRODUCER); - LOGGER.trace("Publisher role is enabled for: {}", configPath); + roles.add(GrpcRole.PUBLISHER); + LOGGER.trace( + "Publisher/Producer role is enabled for: {} (both PRODUCER and PUBLISHER for TQE2/TQE3" + + " compatibility)", + configPath); } final Optional isConsumer = config.getConsumer().flatMap(ConsumerConfig::getEnabled); diff --git a/testcontainers/src/main/java/org/testcontainers/containers/tqe/TQE2ClusterImpl.java b/testcontainers/src/main/java/org/testcontainers/containers/tqe/TQE2ClusterImpl.java new file mode 100644 index 00000000..14b1788d --- /dev/null +++ b/testcontainers/src/main/java/org/testcontainers/containers/tqe/TQE2ClusterImpl.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY + * All Rights Reserved. + */ + +package org.testcontainers.containers.tqe; + +import org.testcontainers.containers.tqe.configuration.TQEConfigurator; + +/** + * Implementation of TQE cluster interface specifically designed for TQE 2.x compatibility. Uses the + * default startup order: queue → configure → grpc. + */ +public class TQE2ClusterImpl extends AbstractTQECluster { + + public TQE2ClusterImpl(TQEConfigurator configurator) { + super(configurator); + } +} diff --git a/testcontainers/src/main/java/org/testcontainers/containers/tqe/TQE3ClusterImpl.java b/testcontainers/src/main/java/org/testcontainers/containers/tqe/TQE3ClusterImpl.java new file mode 100644 index 00000000..ee77c256 --- /dev/null +++ b/testcontainers/src/main/java/org/testcontainers/containers/tqe/TQE3ClusterImpl.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY + * All Rights Reserved. + */ + +package org.testcontainers.containers.tqe; + +import org.testcontainers.containers.tqe.configuration.TQEConfigurator; + +/** + * Implementation of TQE cluster interface specifically designed for TQE 3.x compatibility. Unlike + * TQE 2.x, TQE 3.x configures the cluster between starting queue and gRPC containers: queue → + * configure → grpc. + */ +public class TQE3ClusterImpl extends AbstractTQECluster { + + public TQE3ClusterImpl(TQEConfigurator configurator) { + super(configurator); + } + + @Override + protected void startTarantoolCluster() { + startParallel(this.configurator.queue(), this.configurator); + } + + @Override + protected void startGrpcEndpoints() { + if (!this.configurator.isConfigured()) { + this.configurator.configure(); + } + startParallel(this.configurator.grpc(), this.configurator); + } +} diff --git a/testcontainers/src/main/java/org/testcontainers/containers/tqe/configuration/FileTQEConfigurator.java b/testcontainers/src/main/java/org/testcontainers/containers/tqe/configuration/FileTQEConfigurator.java index 49defe6e..8af8e73c 100644 --- a/testcontainers/src/main/java/org/testcontainers/containers/tqe/configuration/FileTQEConfigurator.java +++ b/testcontainers/src/main/java/org/testcontainers/containers/tqe/configuration/FileTQEConfigurator.java @@ -47,8 +47,8 @@ import io.tarantool.autogen.iproto.listen.Listen; /** - * Base implementation of {@link TQEConfigurator} that configure TQE cluster using configuration - * files. All other implementations should extend this class. + * Configures TQE cluster using configuration files. Supports both TQE 2.x and TQE 3.x via factory + * methods {@link #tqe2Builder} and {@link #tqe3Builder}. */ public class FileTQEConfigurator implements TQEConfigurator { @@ -64,11 +64,12 @@ public class FileTQEConfigurator implements TQEConfigurator { /* Constants /********************************************************** */ + private static final String TQE2_ROUTER_ROLE = "app.roles.api"; + private static final String TQE3_ROUTER_ROLE = "roles.tqe-router"; + private static final String CONFIGURATOR_ERROR_MSG = "An error occurred when configuring the TQE cluster. See logs for details."; - private static final String TQE_ROUTER_ROLE = "roles.tqe-router"; - /* /********************************************************** /* Parameter extractors @@ -87,15 +88,17 @@ public class FileTQEConfigurator implements TQEConfigurator { private final Duration bootstrapTimeout; - private final Map> queue; + private Map> queue; - private final Map> grpc; + private Map> grpc; private final Set routerNames; - private final Tarantool3Configuration queueParsedConfig; + private Tarantool3Configuration queueParsedConfig; - private final Network network; + private Network network; + + private final String routerRole; private boolean configured; @@ -105,79 +108,75 @@ private FileTQEConfigurator( Collection grpcConfigs, String clusterName, Duration startupTimeout, - Duration bootstrapTimeout) { + Duration bootstrapTimeout, + String routerRole) { this.queueConfig = queueConfig; this.grpcConfigs = grpcConfigs; this.clusterName = clusterName; this.bootstrapTimeout = bootstrapTimeout; + this.routerRole = routerRole; this.routerNames = new LinkedHashSet<>(1); this.image = image; this.startupTimeout = startupTimeout; - this.network = Network.newNetwork(); - this.queueParsedConfig = ConfigurationUtils.readFromFile(this.queueConfig); - this.queue = initQueue(this.network, this.image, this.clusterName, this.startupTimeout); - this.grpc = - initGrpc( - this.grpcConfigs, - this.network, - this.image, - this.clusterName, - this.queue, - this.startupTimeout); } - private Map> initGrpc( - Collection grpcConfigs, - Network network, - DockerImageName image, - String clusterName, - Map> queue, - Duration startupTimeout) { + private synchronized Tarantool3Configuration parsedConfig() { + if (this.queueParsedConfig == null) { + this.queueParsedConfig = ConfigurationUtils.readFromFile(this.queueConfig); + } + return this.queueParsedConfig; + } + + private synchronized Network setupNetwork() { + if (this.network == null) { + this.network = Network.newNetwork(); + } + return this.network; + } + + private Map> createGrpcContainers() { Map> grpc = new LinkedHashMap<>(); int i = 1; - for (Path grpcConfig : grpcConfigs) { + for (Path grpcConfig : this.grpcConfigs) { final String grpcName = "grpc-" + i; - final String containerName = grpcName + "-" + clusterName; + final String containerName = grpcName + "-" + this.clusterName; GrpcContainerImpl container = - new GrpcContainerImpl(image, grpcConfig, grpcName, startupTimeout) - .withNetwork(network) + new GrpcContainerImpl(this.image, grpcConfig, grpcName, this.startupTimeout) + .withNetwork(setupNetwork()) .withCreateContainerCmdModifier(cmd -> cmd.withName(containerName)) .withNetworkAliases(containerName) - .dependsOn(queue.values()); + .dependsOn(queue().values()); grpc.put(grpcName, container); } return grpc; } - private Map> initQueue( - Network network, DockerImageName image, String clusterName, Duration startupTimeout) { + private Map> createQueueContainers() { - final List instances = ConfigurationUtils.parseInstances(this.queueParsedConfig); + final List instances = ConfigurationUtils.parseInstances(parsedConfig()); final Map> nodes = new LinkedHashMap<>(instances.size()); this.routerNames.addAll( - ConfigurationUtils.findInstancesWithRole(this.queueParsedConfig, TQE_ROUTER_ROLE)); + ConfigurationUtils.findInstancesWithRole(parsedConfig(), this.routerRole)); if (this.routerNames.isEmpty()) { - LOGGER.error("At least one container must have the 'router' and '{}' roles", TQE_ROUTER_ROLE); + LOGGER.error("At least one container must have the 'router' and '{}' roles", this.routerRole); throw new ContainerLaunchException(CONFIGURATOR_ERROR_MSG); } - final Map advertiseClientUris = - resolveAdvertiseClientUris(this.queueParsedConfig); - final Map advertisePeerUris = - resolveAdvertisePeerUris(this.queueParsedConfig); - final Map> listenUri = resolveListenUris(this.queueParsedConfig); + final Map advertiseClientUris = resolveAdvertiseClientUris(parsedConfig()); + final Map advertisePeerUris = resolveAdvertisePeerUris(parsedConfig()); + final Map> listenUri = resolveListenUris(parsedConfig()); if (listenUri.size() != instances.size()) { LOGGER.error("All nodes should have 'listen.uri' parameter!"); throw new ContainerLaunchException(CONFIGURATOR_ERROR_MSG); } for (String instance : instances) { - final String containerName = instance + "-" + clusterName; + final String containerName = instance + "-" + this.clusterName; final Tarantool3Container container = - new Tarantool3Container(image, instance) - .withNetwork(network) - .withConfigPath(queueConfig) + new Tarantool3Container(this.image, instance) + .withNetwork(setupNetwork()) + .withConfigPath(this.queueConfig) .withNetworkAliases(containerName) .withCreateContainerCmdModifier(cmd -> cmd.withUser("root").withName(containerName)) .waitingFor( @@ -185,7 +184,7 @@ private Map> initQueue( instance, TCMConfig.DEFAULT_TARANTOOL_USERNAME, TCMConfig.DEFAULT_TARANTOOL_PASSWORD)) - .withStartupTimeout(startupTimeout) + .withStartupTimeout(this.startupTimeout) .withCommand("tarantool"); final HostPort advertiseClientUri = advertiseClientUris.get(instance); @@ -378,19 +377,25 @@ public String clusterName() { } @Override - public Map> queue() { + public synchronized Map> queue() { + if (this.queue == null) { + this.queue = createQueueContainers(); + } return this.queue; } @Override - public Map> grpc() { + public synchronized Map> grpc() { + if (this.grpc == null) { + this.grpc = createGrpcContainers(); + } return this.grpc; } @Override public synchronized void configure() { final Entry> router = - this.queue.entrySet().stream() + queue().entrySet().stream() .filter(e -> this.routerNames.contains(e.getKey())) .findFirst() .orElseThrow( @@ -400,13 +405,13 @@ public synchronized void configure() { }); final HostPort hostForRouter = - resolveAdvertiseClientUris(this.queueParsedConfig).entrySet().stream() + resolveAdvertiseClientUris(parsedConfig()).entrySet().stream() .filter(e -> Objects.equals(e.getKey(), router.getKey())) .map(Entry::getValue) .findFirst() .orElseGet( () -> - resolveListenUris(this.queueParsedConfig).entrySet().stream() + resolveListenUris(parsedConfig()).entrySet().stream() .filter(e -> Objects.equals(e.getKey(), router.getKey())) .filter(e -> !e.getValue().isEmpty()) .map(e -> e.getValue().get(0)) @@ -429,29 +434,64 @@ public synchronized boolean isConfigured() { @Override public synchronized void close() { - queue().values().parallelStream().forEach(Startable::close); - grpc().values().parallelStream().forEach(Startable::close); + if (this.queue != null) { + this.queue.values().parallelStream().forEach(Startable::close); + } + if (this.grpc != null) { + this.grpc.values().parallelStream().forEach(Startable::close); + } if (this.network != null) { this.network.close(); } } - public static Builder builder( + /** + * Creates a builder pre-configured for TQE 2.x (router role: {@value #TQE2_ROUTER_ROLE}). + * + * @param image Docker image name + * @param queueConfig path to queue configuration file + * @param grpcConfigs paths to gRPC configuration files + * @return builder with TQE 2.x router role pre-set + */ + public static Builder tqe2Builder( + DockerImageName image, Path queueConfig, Collection grpcConfigs) { + return builder(image, queueConfig, grpcConfigs).withRouterRole(TQE2_ROUTER_ROLE); + } + + /** + * Creates a builder pre-configured for TQE 3.x (router role: {@value #TQE3_ROUTER_ROLE}). + * + * @param image Docker image name + * @param queueConfig path to queue configuration file + * @param grpcConfigs paths to gRPC configuration files + * @return builder with TQE 3.x router role pre-set + */ + public static Builder tqe3Builder( + DockerImageName image, Path queueConfig, Collection grpcConfigs) { + return builder(image, queueConfig, grpcConfigs).withRouterRole(TQE3_ROUTER_ROLE); + } + + private static Builder builder( DockerImageName image, Path queueConfig, Collection grpcConfigs) { return new Builder(image, queueConfig, grpcConfigs); } + /** + * Builder for {@link FileTQEConfigurator}. Use factory methods {@link #tqe2Builder} or {@link + * #tqe3Builder} to obtain a pre-configured builder. + */ public static class Builder { private static final String DEFAULT_CLUSTER_NAME_PREFIX = "tqe-test"; + private final DockerImageName image; private final Path queueConfig; private final Collection grpcConfigs; - private final DockerImageName image; private String clusterName; private Duration startupTimeout; private Duration bootstrapTimeout; + private String routerRole; - public Builder(DockerImageName image, Path queueConfig, Collection grpcConfigs) { + private Builder(DockerImageName image, Path queueConfig, Collection grpcConfigs) { if (queueConfig == null || !Files.isRegularFile(queueConfig)) { LOGGER.error("Queue config file is invalid (null or not regular): {})", queueConfig); throw new IllegalArgumentException(CONFIGURATOR_ERROR_MSG); @@ -474,18 +514,13 @@ public Builder(DockerImageName image, Path queueConfig, Collection grpcCon this.grpcConfigs = new LinkedHashSet<>(grpcConfigs); } + public Builder withRouterRole(String routerRole) { + this.routerRole = Objects.requireNonNull(routerRole, "routerRole must not be null"); + return this; + } + public Builder withClusterName(String clusterName) { - final String defaultClusterName; - if (clusterName == null || clusterName.trim().isEmpty()) { - defaultClusterName = DEFAULT_CLUSTER_NAME_PREFIX + "-" + UUID.randomUUID(); - LOGGER.warn( - "Cluster name is invalid (null or blank): {}. Set default cluster name: '{}'", - clusterName, - defaultClusterName); - } else { - defaultClusterName = clusterName; - } - this.clusterName = defaultClusterName; + this.clusterName = clusterName; return this; } @@ -508,13 +543,19 @@ public FileTQEConfigurator build() { this.startupTimeout == null ? DEFAULT_STARTUP_TIMEOUT : this.startupTimeout; final Duration bootstrapTimeout = this.bootstrapTimeout == null ? DEFAULT_BOOTSTRAP_TIMEOUT : this.bootstrapTimeout; + if (this.routerRole == null) { + throw new IllegalStateException( + "routerRole must be set. Use tqe2Builder() or tqe3Builder() factory method, " + + "or call withRouterRole() explicitly."); + } return new FileTQEConfigurator( this.image, this.queueConfig, this.grpcConfigs, clusterName, startupTimeout, - bootstrapTimeout); + bootstrapTimeout, + this.routerRole); } } } diff --git a/testcontainers/src/main/java/org/testcontainers/containers/tqe/configuration/grpc/GrpcConfiguration.java b/testcontainers/src/main/java/org/testcontainers/containers/tqe/configuration/grpc/GrpcConfiguration.java index 0594fd3d..4604e40f 100644 --- a/testcontainers/src/main/java/org/testcontainers/containers/tqe/configuration/grpc/GrpcConfiguration.java +++ b/testcontainers/src/main/java/org/testcontainers/containers/tqe/configuration/grpc/GrpcConfiguration.java @@ -8,6 +8,7 @@ import java.util.Optional; import java.util.Set; +import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -73,6 +74,7 @@ public class GrpcConfiguration { private final Boolean daemon; @JsonProperty("producer") + @JsonAlias("publisher") private final ProducerConfig producer; @JsonProperty("consumer") diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQETest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQETest.java new file mode 100644 index 00000000..2e3c9dfe --- /dev/null +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQETest.java @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY + * All Rights Reserved. + */ + +package org.testcontainers.containers.integration.tqe; + +import java.net.InetSocketAddress; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import org.junit.jupiter.api.io.TempDir; +import org.rnorth.ducttape.unreliables.Unreliables; +import org.testcontainers.utility.DockerImageName; + +abstract class AbstractTQETest { + + @TempDir protected static Path TEST_TEMP_DIR; + + protected abstract DockerImageName imageName(); + + protected abstract Path simpleGrpcConfig(); + + protected abstract Path simpleQueueConfig(); + + protected static ManagedChannel createReadyChannel(InetSocketAddress address) { + return Unreliables.retryUntilSuccess( + 60, + TimeUnit.SECONDS, + () -> { + ManagedChannel ch = + ManagedChannelBuilder.forAddress(address.getHostName(), address.getPort()) + .usePlaintext() + .maxInboundMessageSize(16 * 1024 * 1024) + .keepAliveTime(30, TimeUnit.SECONDS) + .keepAliveTimeout(5, TimeUnit.SECONDS) + .keepAliveWithoutCalls(true) + .build(); + + ch.getState(true); + Unreliables.retryUntilTrue( + 5, + TimeUnit.SECONDS, + () -> { + io.grpc.ConnectivityState state = ch.getState(false); + if (state == io.grpc.ConnectivityState.READY) { + return true; + } + ch.resetConnectBackoff(); + Thread.sleep(100); + return false; + }); + return ch; + }); + } + + protected static List invalidGrpcConfigs(String producerRoleName) { + return Arrays.asList( + // unknown host + """ + core_port: 1111 + grpc_listen: + - uri: 'tcp://0.0.0.0:18182' + + %s: + enabled: true + tarantool: + user: test-super + pass: test + connections: + routers: + - "unknown:3301" + + consumer: + enabled: true + tarantool: + user: test-super + pass: test + connections: + storage: + - "master:3301" + """ + .formatted(producerRoleName), + // no consumers and producers + """ + core_port: 1111 + grpc_listen: + - uri: 'tcp://0.0.0.0:18182' + + %s: + enabled: false + tarantool: + user: test-super + pass: test + connections: + routers: + - "router:3301" + + consumer: + enabled: false + tarantool: + user: test-super + pass: test + connections: + storage: + - "master:3301" + """ + .formatted(producerRoleName), + // no core_port parameter + """ + grpc_listen: + - uri: 'tcp://0.0.0.0:18182' + + %s: + enabled: true + tarantool: + user: test-super + pass: test + connections: + routers: + - "router:3301" + + consumer: + enabled: true + tarantool: + user: test-super + pass: test + connections: + storage: + - "master:3301" + """ + .formatted(producerRoleName), + // no listen.uri parameter + """ + core_port: 1111 + + %s: + enabled: true + tarantool: + user: test-super + pass: test + connections: + routers: + - "router:3301" + + consumer: + enabled: true + tarantool: + user: test-super + pass: test + connections: + storage: + - "master:3301" + """ + .formatted(producerRoleName)); + } +} diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/CommonTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/CommonTest.java deleted file mode 100644 index 134d35e6..00000000 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/CommonTest.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY - * All Rights Reserved. - */ - -package org.testcontainers.containers.integration.tqe; - -import java.net.URISyntaxException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Objects; - -import org.junit.jupiter.api.io.TempDir; -import org.testcontainers.utility.DockerImageName; - -public abstract class CommonTest { - - @TempDir protected static Path TEST_TEMP_DIR; - - protected static final DockerImageName IMAGE_NAME = - DockerImageName.parse( - System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") - + "tarantool/message-queue-ee:v3.5.0"); - - protected static final Path SIMPLE_GRPC_CONFIG; - protected static final Path SIMPLE_QUEUE_CONFIG; - - static { - try { - SIMPLE_GRPC_CONFIG = - Paths.get( - Objects.requireNonNull( - CommonTest.class - .getClassLoader() - .getResource("tqe/simple-config/simple-grpc.yml")) - .toURI()); - SIMPLE_QUEUE_CONFIG = - Paths.get( - Objects.requireNonNull( - CommonTest.class - .getClassLoader() - .getResource("tqe/simple-config/simple-queue.yml")) - .toURI()); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } -} diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQEConfiguratorTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE2ConfiguratorTest.java similarity index 76% rename from testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQEConfiguratorTest.java rename to testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE2ConfiguratorTest.java index 25fb4c07..f9ae15d3 100644 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQEConfiguratorTest.java +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE2ConfiguratorTest.java @@ -6,8 +6,11 @@ package org.testcontainers.containers.integration.tqe; import java.io.IOException; +import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.stream.Stream; @@ -21,16 +24,61 @@ import org.testcontainers.containers.tqe.configuration.FileTQEConfigurator; import org.testcontainers.containers.tqe.configuration.TQEConfigurator; import org.testcontainers.lifecycle.Startable; +import org.testcontainers.utility.DockerImageName; -class FileTQEConfiguratorTest extends CommonTest { +class FileTQE2ConfiguratorTest extends AbstractTQETest { + + private static final DockerImageName IMAGE_NAME = + DockerImageName.parse( + System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") + + "tarantool/message-queue-ee:2.5.3"); + + private static final Path SIMPLE_GRPC_CONFIG; + private static final Path SIMPLE_QUEUE_CONFIG; + + static { + try { + SIMPLE_GRPC_CONFIG = + Paths.get( + Objects.requireNonNull( + FileTQE2ConfiguratorTest.class + .getClassLoader() + .getResource("tqe2/simple-config/simple-grpc.yml")) + .toURI()); + SIMPLE_QUEUE_CONFIG = + Paths.get( + Objects.requireNonNull( + FileTQE2ConfiguratorTest.class + .getClassLoader() + .getResource("tqe2/simple-config/simple-queue.yml")) + .toURI()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + @Override + protected DockerImageName imageName() { + return IMAGE_NAME; + } + + @Override + protected Path simpleGrpcConfig() { + return SIMPLE_GRPC_CONFIG; + } + + @Override + protected Path simpleQueueConfig() { + return SIMPLE_QUEUE_CONFIG; + } @Test void simpleConfiguration() { try (TQEConfigurator configurator = - FileTQEConfigurator.builder(IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(SIMPLE_GRPC_CONFIG)) + FileTQEConfigurator.tqe2Builder( + imageName(), simpleQueueConfig(), Set.of(simpleGrpcConfig())) .build()) { configurator.queue().values().parallelStream().forEach(Startable::start); - configurator.configure(); configurator.grpc().values().parallelStream().forEach(Startable::start); } catch (Exception e) { throw new RuntimeException(e); @@ -178,8 +226,11 @@ void testInvalidQueueConfig(String invalidQueueConfig) throws IOException { ContainerLaunchException.class, () -> { try (FileTQEConfigurator c = - FileTQEConfigurator.builder(IMAGE_NAME, invalidConfigPath, Set.of(SIMPLE_GRPC_CONFIG)) - .build()) {} + FileTQEConfigurator.tqe2Builder( + imageName(), invalidConfigPath, Set.of(simpleGrpcConfig())) + .build()) { + c.queue(); + } }); } @@ -205,7 +256,7 @@ void testInvalidConfigsPaths(Path invalidGrpcConfig, Set invalidQueueConfi IllegalArgumentException.class, () -> { try (FileTQEConfigurator c = - FileTQEConfigurator.builder(IMAGE_NAME, invalidGrpcConfig, invalidQueueConfigs) + FileTQEConfigurator.tqe2Builder(imageName(), invalidGrpcConfig, invalidQueueConfigs) .build()) {} }); } diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE3ConfiguratorTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE3ConfiguratorTest.java new file mode 100644 index 00000000..1a24870c --- /dev/null +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE3ConfiguratorTest.java @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY + * All Rights Reserved. + */ + +package org.testcontainers.containers.integration.tqe; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.testcontainers.containers.ContainerLaunchException; +import org.testcontainers.containers.tqe.configuration.FileTQEConfigurator; +import org.testcontainers.containers.tqe.configuration.TQEConfigurator; +import org.testcontainers.lifecycle.Startable; +import org.testcontainers.utility.DockerImageName; + +class FileTQE3ConfiguratorTest extends AbstractTQETest { + + private static final DockerImageName IMAGE_NAME = + DockerImageName.parse( + System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") + + "tarantool/message-queue-ee:v3.5.0"); + + private static final Path SIMPLE_GRPC_CONFIG; + private static final Path SIMPLE_QUEUE_CONFIG; + + static { + try { + SIMPLE_GRPC_CONFIG = + Paths.get( + Objects.requireNonNull( + FileTQE3ConfiguratorTest.class + .getClassLoader() + .getResource("tqe3/simple-config/simple-grpc.yml")) + .toURI()); + SIMPLE_QUEUE_CONFIG = + Paths.get( + Objects.requireNonNull( + FileTQE3ConfiguratorTest.class + .getClassLoader() + .getResource("tqe3/simple-config/simple-queue.yml")) + .toURI()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + @Override + protected DockerImageName imageName() { + return IMAGE_NAME; + } + + @Override + protected Path simpleGrpcConfig() { + return SIMPLE_GRPC_CONFIG; + } + + @Override + protected Path simpleQueueConfig() { + return SIMPLE_QUEUE_CONFIG; + } + + @Test + void simpleConfiguration() { + try (TQEConfigurator configurator = + FileTQEConfigurator.tqe3Builder( + imageName(), simpleQueueConfig(), Set.of(simpleGrpcConfig())) + .build()) { + configurator.queue().values().parallelStream().forEach(Startable::start); + configurator.configure(); + configurator.grpc().values().parallelStream().forEach(Startable::start); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static Stream dataForTestInvalidQueueConfigShouldThrow() { + return Stream.of( + // router have no required roles + """ + # Credentials + credentials: + users: + test-super: + password: 'test' + roles: [ super ] + admin: + password: 'secret-cluster-cookie' + roles: [ super ] + replicator: + password: 'secret' + roles: [ replication ] + storage: + roles: [ sharding ] + password: storage + # advertise configs for all nodes + iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + password: storage + roles: [ roles.metrics-export ] + # queues configs + roles_cfg: + roles.tqe-storage: + queues: + - name: test + deduplication_mode: keep_latest + disabled_filters_by: [ sharding_key ] + roles.metrics-export: + http: + - listen: 8081 + endpoints: + - format: prometheus + path: '/metrics' + groups: + routers: + replicasets: + router-1: + sharding: + roles: [ router ] + instances: + router: + iproto: + listen: + - uri: router:3301 + storages: + replicasets: + storage-1: + replication: + failover: manual + sharding: + roles: [ storage ] + roles: + - roles.tqe-storage + leader: master + instances: + master: + iproto: + listen: + - uri: master:3301 + """, + """ + # Credentials + credentials: + users: + test-super: + password: 'test' + roles: [ super ] + admin: + password: 'secret-cluster-cookie' + roles: [ super ] + replicator: + password: 'secret' + roles: [ replication ] + storage: + roles: [ sharding ] + password: storage + # advertise configs for all nodes + iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + password: storage + roles: [ roles.metrics-export ] + # queues configs + roles_cfg: + roles.tqe-storage: + queues: + - name: test + deduplication_mode: keep_latest + disabled_filters_by: [ sharding_key ] + roles.metrics-export: + http: + - listen: 8081 + endpoints: + - format: prometheus + path: '/metrics' + groups: + routers: + replicasets: + router-1: + sharding: + roles: [ router ] + roles: + - roles.tqe-router + instances: + router: + iproto: + listen: + - uri: router:3301 + storages: + replicasets: + storage-1: + replication: + failover: manual + sharding: + roles: [ storage ] + leader: master + instances: + master: + iproto: + net_msg_max: 768 + """); + } + + @ParameterizedTest + @MethodSource("dataForTestInvalidQueueConfigShouldThrow") + void testInvalidQueueConfig(String invalidQueueConfig) throws IOException { + final Path invalidConfigPath = TEST_TEMP_DIR.resolve(UUID.randomUUID().toString()); + Files.writeString(invalidConfigPath, invalidQueueConfig); + + Assertions.assertThrows( + ContainerLaunchException.class, + () -> { + try (FileTQEConfigurator c = + FileTQEConfigurator.tqe3Builder( + imageName(), invalidConfigPath, Set.of(simpleGrpcConfig())) + .build()) { + c.queue(); + } + }); + } + + public static Stream dataForTestInvalidConfigsPaths() { + return Stream.of( + // invalid grpc configs + // null + Arguments.of(SIMPLE_QUEUE_CONFIG, null), + // empty + Arguments.of(SIMPLE_QUEUE_CONFIG, Set.of()), + // non regular + Arguments.of(SIMPLE_QUEUE_CONFIG, Set.of(TEST_TEMP_DIR)), + + // invalid queue config + Arguments.of(null, Set.of(SIMPLE_GRPC_CONFIG)), + Arguments.of(TEST_TEMP_DIR, Set.of(SIMPLE_GRPC_CONFIG))); + } + + @ParameterizedTest + @MethodSource("dataForTestInvalidConfigsPaths") + void testInvalidConfigsPaths(Path invalidGrpcConfig, Set invalidQueueConfigs) { + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + try (FileTQEConfigurator c = + FileTQEConfigurator.tqe3Builder(imageName(), invalidGrpcConfig, invalidQueueConfigs) + .build()) {} + }); + } +} diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java new file mode 100644 index 00000000..8017995c --- /dev/null +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java @@ -0,0 +1,429 @@ +/* + * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY + * All Rights Reserved. + */ + +package org.testcontainers.containers.integration.tqe; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CopyOnWriteArraySet; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import com.google.protobuf.ByteString; +import io.grpc.ManagedChannel; +import io.grpc.stub.StreamObserver; +import org.instancio.Instancio; +import org.instancio.Select; +import org.instancio.generators.Generators; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.RepeatedTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.rnorth.ducttape.unreliables.Unreliables; +import org.testcontainers.containers.ContainerLaunchException; +import org.testcontainers.containers.tqe.GrpcContainer; +import org.testcontainers.containers.tqe.GrpcContainer.GrpcRole; +import org.testcontainers.containers.tqe.TQE2ClusterImpl; +import org.testcontainers.containers.tqe.TQECluster; +import org.testcontainers.containers.tqe.configuration.FileTQEConfigurator; +import org.testcontainers.containers.tqe.configuration.TQEConfigurator; +import org.testcontainers.containers.utils.pojo.User; +import org.testcontainers.shaded.com.fasterxml.jackson.databind.ObjectMapper; +import org.testcontainers.utility.DockerImageName; +import tarantool.queue_ee.v2.Consumer.SubscriptionNotifications; +import tarantool.queue_ee.v2.Consumer.SubscriptionRequest; +import tarantool.queue_ee.v2.ConsumerServiceGrpc; +import tarantool.queue_ee.v2.ConsumerServiceGrpc.ConsumerServiceStub; +import tarantool.queue_ee.v2.Publisher.BatchRequestMessage; +import tarantool.queue_ee.v2.Publisher.PublishBatchRequest; +import tarantool.queue_ee.v2.PublisherServiceGrpc; +import tarantool.queue_ee.v2.PublisherServiceGrpc.PublisherServiceBlockingStub; + +class TQE2ClusterImplTest extends AbstractTQETest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private static final DockerImageName IMAGE_NAME = + DockerImageName.parse( + System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") + + "tarantool/message-queue-ee:2.5.3"); + + private static final Path SIMPLE_GRPC_CONFIG; + private static final Path SIMPLE_QUEUE_CONFIG; + + static { + try { + SIMPLE_GRPC_CONFIG = + Paths.get( + Objects.requireNonNull( + TQE2ClusterImplTest.class + .getClassLoader() + .getResource("tqe2/simple-config/simple-grpc.yml")) + .toURI()); + SIMPLE_QUEUE_CONFIG = + Paths.get( + Objects.requireNonNull( + TQE2ClusterImplTest.class + .getClassLoader() + .getResource("tqe2/simple-config/simple-queue.yml")) + .toURI()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + @Override + protected DockerImageName imageName() { + return IMAGE_NAME; + } + + @Override + protected Path simpleGrpcConfig() { + return SIMPLE_GRPC_CONFIG; + } + + @Override + protected Path simpleQueueConfig() { + return SIMPLE_QUEUE_CONFIG; + } + + @RepeatedTest(10) + void testMultiplyRestart() throws Exception { + try (TQEConfigurator configurator = + FileTQEConfigurator.tqe2Builder( + IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(SIMPLE_GRPC_CONFIG)) + .build(); + TQECluster cluster = new TQE2ClusterImpl(configurator)) { + cluster.start(); + } + } + + @Test + void testRestartMethod() throws Exception { + try (TQEConfigurator configurator = + FileTQEConfigurator.tqe2Builder( + IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(SIMPLE_GRPC_CONFIG)) + .build(); + TQECluster cluster = new TQE2ClusterImpl(configurator)) { + cluster.start(); + cluster.restart(1, TimeUnit.SECONDS, 1, TimeUnit.SECONDS); + } + } + + public static Stream dataForTestInvalidQueueConfigShouldThrow() { + final List invalidConfigs = + Arrays.asList( + // no required test-super user + """ + # Credentials + credentials: + users: + admin: + password: 'secret-cluster-cookie' + roles: [ super ] + replicator: + password: 'secret' + roles: [ replication ] + storage: + roles: [ sharding ] + password: storage + + # advertise configs for all nodes + iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + password: storage + + roles: [ roles.metrics-export ] + # queues configs + roles_cfg: + app.roles.queue: + queues: + - name: test + deduplication_mode: keep_latest + disabled_filters_by: [ sharding_key ] + roles.metrics-export: + http: + - listen: 8081 + endpoints: + - format: prometheus + path: '/metrics' + + groups: + routers: + replicasets: + r-1: + sharding: + roles: [ router ] + roles: [ app.roles.api ] + instances: + router: + iproto: + listen: + - uri: router:3301 + storages: + replicasets: + shard-1: + replication: + failover: manual + sharding: + roles: [ storage ] + leader: master + instances: + master: + iproto: + listen: + - uri: master:3301 + net_msg_max: 768 + """, + // no consumer storage to connect from grpc + """ + # Credentials + credentials: + users: + test-super: + password: 'test' + roles: [ super ] + admin: + password: 'secret-cluster-cookie' + roles: [ super ] + replicator: + password: 'secret' + roles: [ replication ] + storage: + roles: [ sharding ] + password: storage + + # advertise configs for all nodes + iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + password: storage + + roles: [ roles.metrics-export ] + # queues configs + roles_cfg: + app.roles.queue: + queues: + - name: test + deduplication_mode: keep_latest + disabled_filters_by: [ sharding_key ] + roles.metrics-export: + http: + - listen: 8081 + endpoints: + - format: prometheus + path: '/metrics' + + groups: + routers: + replicasets: + r-1: + sharding: + roles: [ router ] + roles: [ app.roles.api ] + instances: + router: + iproto: + listen: + - uri: router:3301 + """); + + return invalidConfigs.stream() + .map( + s -> { + final Path testConfigPath = TEST_TEMP_DIR.resolve(UUID.randomUUID().toString()); + try { + Files.writeString(testConfigPath, s); + return Arguments.of(testConfigPath); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + @ParameterizedTest + @MethodSource("dataForTestInvalidQueueConfigShouldThrow") + void testInvalidQueueConfigShouldThrow(Path queueConfig) { + Assertions.assertThrows( + ContainerLaunchException.class, + () -> { + try (TQEConfigurator configurator = + FileTQEConfigurator.tqe2Builder( + IMAGE_NAME, queueConfig, Set.of(SIMPLE_GRPC_CONFIG)) + .withStartupTimeout(Duration.ofSeconds(5)) + .build(); + TQECluster cluster = new TQE2ClusterImpl(configurator)) { + cluster.start(); + } + }); + } + + public static Stream dataForTestInvalidGrpcConfig() { + return invalidGrpcConfigs("publisher").stream() + .map( + s -> { + final Path testConfigPath = TEST_TEMP_DIR.resolve(UUID.randomUUID() + ".yml"); + try { + Files.writeString(testConfigPath, s); + return testConfigPath; + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + @ParameterizedTest + @MethodSource("dataForTestInvalidGrpcConfig") + void testInvalidGrpcConfig(Path grpcConfig) { + Assertions.assertThrows( + ContainerLaunchException.class, + () -> { + try (TQEConfigurator configurator = + FileTQEConfigurator.tqe2Builder( + IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(grpcConfig)) + .withStartupTimeout(Duration.ofSeconds(5)) + .build(); + TQECluster cluster = new TQE2ClusterImpl(configurator)) { + cluster.start(); + } + }); + } + + @RepeatedTest(10) + void testPublishAndConsumeData() { + Assertions.assertDoesNotThrow( + () -> { + try (TQEConfigurator configurator = + FileTQEConfigurator.tqe2Builder( + IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(SIMPLE_GRPC_CONFIG)) + .build(); + TQECluster cluster = new TQE2ClusterImpl(configurator)) { + cluster.start(); + + final String queueName = "test"; + + final List> publishers = + cluster.grpc().values().stream() + .filter(g -> g.roles().contains(GrpcRole.PUBLISHER)) + .toList(); + final List> consumers = + cluster.grpc().values().stream() + .filter(g -> g.roles().contains(GrpcRole.CONSUMER)) + .toList(); + + Assertions.assertFalse(publishers.isEmpty()); + Assertions.assertFalse(consumers.isEmpty()); + + final Set grpcAddresses = publishers.get(0).grpcAddresses(); + final Set consumerAddresses = consumers.get(0).grpcAddresses(); + + final Optional publisherAddress = grpcAddresses.stream().findFirst(); + Assertions.assertTrue(publisherAddress.isPresent()); + final Optional consumerAddress = + consumerAddresses.stream().findFirst(); + Assertions.assertTrue(consumerAddress.isPresent()); + + final ManagedChannel publisherChannel = createReadyChannel(publisherAddress.get()); + + final ManagedChannel consumerChannel = createReadyChannel(consumerAddress.get()); + + final PublisherServiceBlockingStub publisherService = + PublisherServiceGrpc.newBlockingStub(publisherChannel); + final ConsumerServiceStub consumerService = + ConsumerServiceGrpc.newStub(consumerChannel); + + try { + final List users = + Instancio.ofList(User.class) + .size(100) + .generate( + Select.field(User::getName), + g -> g.string().alphaNumeric().allowEmpty().nullable()) + .generate(Select.field(User::getAge), Generators::ints) + .create(); + + Unreliables.retryUntilSuccess( + 60, + TimeUnit.SECONDS, + () -> { + final PublishBatchRequest.Builder requestBuilder = + PublishBatchRequest.newBuilder(); + for (User user : users) { + requestBuilder.addMessages( + BatchRequestMessage.newBuilder() + .setPayload(ByteString.copyFrom(MAPPER.writeValueAsBytes(user)))); + } + final PublishBatchRequest publishRequest = + requestBuilder.setQueue(queueName).build(); + publisherService.publishBatch(publishRequest); + return true; + }); + + final Set result = new CopyOnWriteArraySet<>(); + Unreliables.retryUntilSuccess( + 60, + TimeUnit.SECONDS, + () -> { + consumerService.subscribe( + SubscriptionRequest.newBuilder().setCursor("").setQueue(queueName).build(), + new StreamObserver<>() { + @Override + public void onNext(SubscriptionNotifications value) { + value.getNotificationsList().stream() + .map( + n -> { + try { + return MAPPER.readValue( + n.getMessage().getPayload().toByteArray(), User.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .forEach(result::add); + } + + @Override + public void onError(Throwable t) { + throw new RuntimeException("Stream observer received error", t); + } + + @Override + public void onCompleted() {} + }); + return true; + }); + + Unreliables.retryUntilTrue( + 60, TimeUnit.SECONDS, () -> new LinkedHashSet<>(users).size() == result.size()); + Assertions.assertEquals(new LinkedHashSet<>(users), result); + } finally { + consumerChannel.shutdownNow(); + publisherChannel.shutdownNow(); + } + } + }); + } +} diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterImplTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java similarity index 57% rename from testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterImplTest.java rename to testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java index 7633380c..9f8985d9 100644 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterImplTest.java +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java @@ -7,12 +7,15 @@ import java.io.IOException; import java.net.InetSocketAddress; +import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.time.Duration; import java.util.Arrays; import java.util.LinkedHashSet; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -22,7 +25,6 @@ import com.google.protobuf.ByteString; import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; import io.grpc.stub.StreamObserver; import org.instancio.Instancio; import org.instancio.Select; @@ -37,12 +39,13 @@ import org.testcontainers.containers.ContainerLaunchException; import org.testcontainers.containers.tqe.GrpcContainer; import org.testcontainers.containers.tqe.GrpcContainer.GrpcRole; +import org.testcontainers.containers.tqe.TQE3ClusterImpl; import org.testcontainers.containers.tqe.TQECluster; -import org.testcontainers.containers.tqe.TQEClusterImpl; import org.testcontainers.containers.tqe.configuration.FileTQEConfigurator; import org.testcontainers.containers.tqe.configuration.TQEConfigurator; import org.testcontainers.containers.utils.pojo.User; import org.testcontainers.shaded.com.fasterxml.jackson.databind.ObjectMapper; +import org.testcontainers.utility.DockerImageName; import tarantool.queue_ee.Consumer.SubscriptionRequest; import tarantool.queue_ee.Consumer.SubscriptionStreamRequest; import tarantool.queue_ee.Consumer.SubscriptionStreamResponse; @@ -53,14 +56,61 @@ import tarantool.queue_ee.ProducerOuterClass.ProduceMessage; import tarantool.queue_ee.ProducerOuterClass.ProduceRequest; -class TQEClusterImplTest extends CommonTest { +class TQE3ClusterImplTest extends AbstractTQETest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private static final DockerImageName IMAGE_NAME = + DockerImageName.parse( + System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") + + "tarantool/message-queue-ee:v3.5.0"); + + private static final Path SIMPLE_GRPC_CONFIG; + private static final Path SIMPLE_QUEUE_CONFIG; + + static { + try { + SIMPLE_GRPC_CONFIG = + Paths.get( + Objects.requireNonNull( + TQE3ClusterImplTest.class + .getClassLoader() + .getResource("tqe3/simple-config/simple-grpc.yml")) + .toURI()); + SIMPLE_QUEUE_CONFIG = + Paths.get( + Objects.requireNonNull( + TQE3ClusterImplTest.class + .getClassLoader() + .getResource("tqe3/simple-config/simple-queue.yml")) + .toURI()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + @Override + protected DockerImageName imageName() { + return IMAGE_NAME; + } + + @Override + protected Path simpleGrpcConfig() { + return SIMPLE_GRPC_CONFIG; + } + + @Override + protected Path simpleQueueConfig() { + return SIMPLE_QUEUE_CONFIG; + } @RepeatedTest(10) void testMultiplyRestart() throws Exception { try (TQEConfigurator configurator = - FileTQEConfigurator.builder(IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(SIMPLE_GRPC_CONFIG)) + FileTQEConfigurator.tqe3Builder( + IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(SIMPLE_GRPC_CONFIG)) .build(); - TQECluster cluster = new TQEClusterImpl(configurator)) { + TQECluster cluster = new TQE3ClusterImpl(configurator)) { cluster.start(); } } @@ -68,9 +118,10 @@ void testMultiplyRestart() throws Exception { @Test void testRestartMethod() throws Exception { try (TQEConfigurator configurator = - FileTQEConfigurator.builder(IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(SIMPLE_GRPC_CONFIG)) + FileTQEConfigurator.tqe3Builder( + IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(SIMPLE_GRPC_CONFIG)) .build(); - TQECluster cluster = new TQEClusterImpl(configurator)) { + TQECluster cluster = new TQE3ClusterImpl(configurator)) { cluster.start(); cluster.restart(1, TimeUnit.SECONDS, 1, TimeUnit.SECONDS); } @@ -106,7 +157,7 @@ public static Stream dataForTestInvalidQueueConfigShouldThrow() { roles: [ roles.metrics-export ] # queues configs roles_cfg: - app.roles.queue: + roles.tqe-storage: queues: - name: test deduplication_mode: keep_latest @@ -121,10 +172,11 @@ public static Stream dataForTestInvalidQueueConfigShouldThrow() { groups: routers: replicasets: - r-1: + router-1: sharding: roles: [ router ] - roles: [ app.roles.api ] + roles: + - roles.tqe-router instances: router: iproto: @@ -132,11 +184,13 @@ public static Stream dataForTestInvalidQueueConfigShouldThrow() { - uri: router:3301 storages: replicasets: - shard-1: + storage-1: replication: failover: manual sharding: roles: [ storage ] + roles: + - roles.tqe-storage leader: master instances: master: @@ -175,7 +229,7 @@ public static Stream dataForTestInvalidQueueConfigShouldThrow() { roles: [ roles.metrics-export ] # queues configs roles_cfg: - app.roles.queue: + roles.tqe-storage: queues: - name: test deduplication_mode: keep_latest @@ -190,10 +244,11 @@ public static Stream dataForTestInvalidQueueConfigShouldThrow() { groups: routers: replicasets: - r-1: + router-1: sharding: roles: [ router ] - roles: [ app.roles.api ] + roles: + - roles.tqe-router instances: router: iproto: @@ -221,112 +276,18 @@ void testInvalidQueueConfigShouldThrow(Path queueConfig) { ContainerLaunchException.class, () -> { try (TQEConfigurator configurator = - FileTQEConfigurator.builder(IMAGE_NAME, queueConfig, Set.of(SIMPLE_GRPC_CONFIG)) + FileTQEConfigurator.tqe3Builder( + IMAGE_NAME, queueConfig, Set.of(SIMPLE_GRPC_CONFIG)) .withStartupTimeout(Duration.ofSeconds(5)) .build(); - TQECluster cluster = new TQEClusterImpl(configurator)) { + TQECluster cluster = new TQE3ClusterImpl(configurator)) { cluster.start(); } }); } public static Stream dataForTestInvalidGrpcConfig() { - final List invalidGrpcConfigs = - Arrays.asList( - """ - core_port: 1111 - grpc_listen: - - uri: 'tcp://0.0.0.0:18182' - - producer: - enabled: true - tarantool: - user: test-super - pass: test - connections: - routers: - - "unknown:3301" - - consumer: - enabled: true - tarantool: - user: test-super - pass: test - connections: - storage: - - "master:3301" - """, - // no consumers and producers - """ - core_port: 1111 - grpc_listen: - - uri: 'tcp://0.0.0.0:18182' - - producer: - enabled: false - tarantool: - user: test-super - pass: test - connections: - routers: - - "router:3301" - - consumer: - enabled: false - tarantool: - user: test-super - pass: test - connections: - storage: - - "master:3301" - """, - // no core_port parameter - """ - grpc_listen: - - uri: 'tcp://0.0.0.0:18182' - - producer: - enabled: true - tarantool: - user: test-super - pass: test - connections: - routers: - - "router:3301" - - consumer: - enabled: true - tarantool: - user: test-super - pass: test - connections: - storage: - - "master:3301" - """, - // no listen.uri parameter - """ - core_port: 1111 - - producer: - enabled: true - tarantool: - user: test-super - pass: test - connections: - routers: - - "router:3301" - - consumer: - enabled: true - tarantool: - user: test-super - pass: test - connections: - storage: - - "master:3301" - """); - - return invalidGrpcConfigs.stream() + return invalidGrpcConfigs("producer").stream() .map( s -> { final Path testConfigPath = TEST_TEMP_DIR.resolve(UUID.randomUUID() + ".yml"); @@ -346,10 +307,11 @@ void testInvalidGrpcConfig(Path grpcConfig) { ContainerLaunchException.class, () -> { try (TQEConfigurator configurator = - FileTQEConfigurator.builder(IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(grpcConfig)) + FileTQEConfigurator.tqe3Builder( + IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(grpcConfig)) .withStartupTimeout(Duration.ofSeconds(5)) .build(); - TQECluster cluster = new TQEClusterImpl(configurator)) { + TQECluster cluster = new TQE3ClusterImpl(configurator)) { cluster.start(); } }); @@ -359,13 +321,11 @@ void testInvalidGrpcConfig(Path grpcConfig) { void testPublishAndConsumeData() { Assertions.assertDoesNotThrow( () -> { - final ObjectMapper MAPPER = new ObjectMapper(); - try (TQEConfigurator configurator = - FileTQEConfigurator.builder( + FileTQEConfigurator.tqe3Builder( IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(SIMPLE_GRPC_CONFIG)) .build(); - TQECluster cluster = new TQEClusterImpl(configurator)) { + TQECluster cluster = new TQE3ClusterImpl(configurator)) { cluster.start(); final String queueName = "test"; @@ -385,82 +345,92 @@ void testPublishAndConsumeData() { final Set grpcAddresses = producers.get(0).grpcAddresses(); final Set consumerAddresses = consumers.get(0).grpcAddresses(); - final Optional publisherAddress = grpcAddresses.stream().findFirst(); - Assertions.assertTrue(publisherAddress.isPresent()); + final Optional producerAddress = grpcAddresses.stream().findFirst(); + Assertions.assertTrue(producerAddress.isPresent()); final Optional consumerAddress = consumerAddresses.stream().findFirst(); Assertions.assertTrue(consumerAddress.isPresent()); - final ManagedChannel producerChannel = - ManagedChannelBuilder.forAddress( - publisherAddress.get().getHostName(), publisherAddress.get().getPort()) - .usePlaintext() - .build(); + final ManagedChannel producerChannel = createReadyChannel(producerAddress.get()); - final ManagedChannel consumerChannel = - ManagedChannelBuilder.forAddress( - consumerAddress.get().getHostName(), consumerAddress.get().getPort()) - .usePlaintext() - .build(); + final ManagedChannel consumerChannel = createReadyChannel(consumerAddress.get()); final ProducerBlockingStub producer = ProducerGrpc.newBlockingStub(producerChannel); final ConsumerServiceStub consumer = ConsumerServiceGrpc.newStub(consumerChannel); - final List users = - Instancio.ofList(User.class) - .size(100) - .generate( - Select.field(User::getName), - g -> g.string().alphaNumeric().allowEmpty().nullable()) - .generate(Select.field(User::getAge), Generators::ints) - .create(); - - final ProduceRequest.Builder requestBuilder = - ProduceRequest.newBuilder().setQueue(queueName); - for (User user : users) { - requestBuilder.addMessages( - ProduceMessage.newBuilder() - .setPayload(ByteString.copyFrom(MAPPER.writeValueAsBytes(user)))); + try { + final List users = + Instancio.ofList(User.class) + .size(100) + .generate( + Select.field(User::getName), + g -> g.string().alphaNumeric().allowEmpty().nullable()) + .generate(Select.field(User::getAge), Generators::ints) + .create(); + + Unreliables.retryUntilSuccess( + 60, + TimeUnit.SECONDS, + () -> { + final ProduceRequest.Builder requestBuilder = + ProduceRequest.newBuilder().setQueue(queueName); + for (User user : users) { + requestBuilder.addMessages( + ProduceMessage.newBuilder() + .setPayload(ByteString.copyFrom(MAPPER.writeValueAsBytes(user)))); + } + final ProduceRequest produceRequest = requestBuilder.build(); + producer.produce(produceRequest); + return true; + }); + + final Set result = new CopyOnWriteArraySet<>(); + Unreliables.retryUntilSuccess( + 60, + TimeUnit.SECONDS, + () -> { + StreamObserver requestsStream = + consumer.subscribe( + new StreamObserver() { + @Override + public void onNext(SubscriptionStreamResponse response) { + response.getNotifications().getNotificationsList().stream() + .map( + n -> { + try { + return MAPPER.readValue( + n.getMessage().getPayload().toByteArray(), + User.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .forEach(result::add); + } + + @Override + public void onError(Throwable t) { + throw new RuntimeException("Stream observer received error", t); + } + + @Override + public void onCompleted() {} + }); + requestsStream.onNext( + SubscriptionStreamRequest.newBuilder() + .setSubscribeRequest( + SubscriptionRequest.newBuilder().setCursor("").setQueue(queueName)) + .build()); + return true; + }); + + Unreliables.retryUntilTrue( + 60, TimeUnit.SECONDS, () -> new LinkedHashSet<>(users).size() == result.size()); + Assertions.assertEquals(new LinkedHashSet<>(users), result); + } finally { + consumerChannel.shutdownNow(); + producerChannel.shutdownNow(); } - final ProduceRequest produceRequest = requestBuilder.build(); - producer.produce(produceRequest); - - final Set result = new CopyOnWriteArraySet<>(); - StreamObserver requestsStream = - consumer.subscribe( - new StreamObserver() { - @Override - public void onNext(SubscriptionStreamResponse response) { - response.getNotifications().getNotificationsList().stream() - .map( - n -> { - try { - return MAPPER.readValue( - n.getMessage().getPayload().toByteArray(), User.class); - } catch (IOException e) { - throw new RuntimeException(e); - } - }) - .forEach(result::add); - } - - @Override - public void onError(Throwable t) {} - - @Override - public void onCompleted() {} - }); - requestsStream.onNext( - SubscriptionStreamRequest.newBuilder() - .setSubscribeRequest( - SubscriptionRequest.newBuilder().setCursor("").setQueue(queueName)) - .build()); - - Unreliables.retryUntilTrue( - 5, TimeUnit.SECONDS, () -> new LinkedHashSet<>(users).size() == result.size()); - Assertions.assertEquals(new LinkedHashSet<>(users), result); - consumerChannel.shutdownNow(); - producerChannel.shutdownNow(); } }); } diff --git a/testcontainers/src/test/proto/tqe2/messages/message.proto b/testcontainers/src/test/proto/tqe2/messages/message.proto new file mode 100644 index 00000000..595f4740 --- /dev/null +++ b/testcontainers/src/test/proto/tqe2/messages/message.proto @@ -0,0 +1,53 @@ +syntax = "proto3"; + +package tarantool.queue_ee; + +option go_package = "gitlab.vkteam.ru/tarantool/tqe/message-queue.git/v2/server/protocol"; +option java_package = "tarantool.queue_ee.v2"; +option java_outer_classname = "Message"; + +// Пара ключ-значение +message Pair { + // Ключ пары + string key = 1; + + // Значение пары + string value = 2; +} + +// Сообщение в очереди +message QueueMessage { + // Идентификатор сообщения + // Заполняется автоматически при записи сообщения в очередь + uint64 id = 1; + + // Название очереди в которую необходимо опубликовать сообщение + string queue = 2; + + // Ключ маршрутизации сообщения (тип сообщения) + // необходим для фильтрации сообщений из очереди на консьюмерах + optional string routing_key = 3; + + // Ключ шардирования + // необходим для распределения данных в системе + optional string sharding_key = 4; + + // Ключ дедупликации + // необходим для проверки повторных сообщений, + // если не указан, то проверка не производится + optional string deduplication_key = 5; + + // Произвольные данные в бинарном формате, содержит тело сообщения + bytes payload = 6; + + // Произвольные данные в бинарном формате, + // содержит дополнительные для сообщения данные, + // необходимые для отладки и трассировки + map metadata = 7 [deprecated = true]; + + // Время вставки сообщения в очередь в наносекундах + int64 timestamp = 8; + + // Произвольные данные в формате списка из пар ключ-значения + repeated Pair metadata_pairs = 9; +} diff --git a/testcontainers/src/test/proto/tqe2/services/consumer.proto b/testcontainers/src/test/proto/tqe2/services/consumer.proto new file mode 100644 index 00000000..dc24e94f --- /dev/null +++ b/testcontainers/src/test/proto/tqe2/services/consumer.proto @@ -0,0 +1,58 @@ +syntax = "proto3"; + +package tarantool.queue_ee; + +import "messages/message.proto"; + +option go_package = "gitlab.vkteam.ru/tarantool/tqe/message-queue.git/v2/server/protocol"; +option java_package = "tarantool.queue_ee.v2"; +option java_outer_classname = "Consumer"; + +// Сервер подписок на сообщения брокера очередей +service ConsumerService { + // Подписка на сообщения с фильтром + rpc Subscribe(SubscriptionRequest) returns (stream SubscriptionNotifications); +} + +// Запрос на подписку +message SubscriptionRequest { + // Название очереди + string queue = 1; + + // Ключ маршрутизации сообщения (тип сообщения) + // необходим для фильтрации сообщений из очереди + // Если не указан, то подписка происходит на все типы сообщений в очереди + optional string routing_key = 2; + + // Опциональная строка указатель на последнее полученное сообщение. + // Необходимо для возможности получения истории сообщений + // или восстановления работы консьюмера после сбоя + // Значение не указано - подписка с текущего момента + // Значение пустая строка - подписка с начала очереди + // Значение указано - подписка с указанного сообщения в очереди + optional string cursor = 3; + + // Ключ шардирования + // необходим для распределения данных в системе + // Если не указан, то подписка происходит на все типы сообщений в очереди + optional string sharding_key = 4; + + // Ключи шардирования позволяют производить фильтрацию по нескольким ключам + // шардирования в рамках одной подписки + repeated string sharding_keys = 5; +} + +// Сообщение в стриме подписки +message SubscriptionNotifications { + // Новые сообщения в очереди с курсорами + repeated SubscriptionNotification notifications = 1; +} + +// Уведомление клиента о новых сообщение в очереди +message SubscriptionNotification { + // Строка-указатель сообщения + string cursor = 1; + + // Сообщение + QueueMessage message = 2; +} diff --git a/testcontainers/src/test/proto/tqe2/services/publisher.proto b/testcontainers/src/test/proto/tqe2/services/publisher.proto new file mode 100644 index 00000000..949e50f2 --- /dev/null +++ b/testcontainers/src/test/proto/tqe2/services/publisher.proto @@ -0,0 +1,263 @@ +syntax = "proto3"; + +package tarantool.queue_ee; + +import "messages/message.proto"; + +option go_package = "gitlab.vkteam.ru/tarantool/tqe/message-queue.git/v2/server/protocol"; +option java_package = "tarantool.queue_ee.v2"; +option java_outer_classname = "Publisher"; + +// Сервер публикации сообщений брокера очередей +service PublisherService { + // Публикация сообщения в очередь + rpc Publish(PublishRequest) returns (PublishResponse); + // Публикация сообщений в очередь через двусторонний стрим + rpc PublishStream(stream PublishStreamRequest) returns (stream PublishStreamResponse); + + // Публикация набора сообщений в очередь + rpc PublishBatch(PublishBatchRequest) returns (PublishBatchResponse); + // Публикация набора сообщений в очередь через двусторонний стрим + rpc PublishBatchStream( + stream PublishBatchStreamRequest + ) returns ( + stream PublishBatchStreamResponse + ); + + // Публикация сообщения на указанные шарды очереди + rpc Broadcast(BroadcastRequest) returns (BroadcastResponse); +} + +// Режим дедупликации сообщения +enum Deduplication { + DEDUPLICATION_UNSPECIFIED = 0; + DEDUPLICATION_BASIC = 1; + DEDUPLICATION_EXTENDED = 2; + DEDUPLICATION_KEEP_FIRST = 3; + DEDUPLICATION_KEEP_LATEST = 4; +} + +// Запрос на публикацию сообщения в очередь +message PublishRequest { + // Название очереди в которой необходимо опубликовать сообщение + string queue = 1; + + // Ключ маршрутизации сообщения (тип сообщения) + // необходим для фильтрации сообщений из очереди на консьюмерах + optional string routing_key = 2; + + // Ключ шардирования + // необходим для распределения данных в системе + optional string sharding_key = 3; + + // Ключ дедупликации + // необходим для проверки повторных сообщений, + // если не указан, то проверка не производится + optional string deduplication_key = 4; + + // Произвольные данные в бинарном формате, содержит тело сообщения + bytes payload = 5; + + // Произвольные данные в бинарном формате, + // содержит дополнительные для сообщения данные, + // необходимые для отладки и трассировки + map metadata = 6 [deprecated = true]; + + // Произвольные данные в формате списка из пар ключ-значения + repeated Pair metadata_pairs = 7; + + // Режим дедупликации сообщения + Deduplication deduplication = 8; +} + +// Запрос на публикация набора сообщений в очередь +message PublishBatchRequest { + // Название очереди в которой необходимо опубликовать сообщения + string queue = 1; + + // Ключ шардирования + // необходим для распределения данных в системе + optional string sharding_key = 2; + + // Набор сообщений + repeated BatchRequestMessage messages = 3; + + // Содержит дополнительные данные необходимые для отладки и трассировки + map metadata = 4 [deprecated = true]; + + // Произвольные данные в формате списка из пар ключ-значения + repeated Pair metadata_pairs = 5; + + // Режим дедупликации сообщения + Deduplication deduplication = 8; +} + +// Набор сообщений +message BatchRequestMessage { + // Ключ маршрутизации сообщения (тип сообщения) + // необходим для фильтрации сообщений из очереди на консьюмерах + optional string routing_key = 1; + + // Ключ дедупликации + // Необходим для проверки повторных сообщений, + // если не указан, то проверка не производится + optional string deduplication_key = 2; + + // Произвольные данные в бинарном формате, содержит тело сообщения + bytes payload = 3; + + // Произвольные данные в бинарном формате, + // содержит дополнительные для сообщения данные, + // необходимые для отладки и трассировки + map metadata = 4 [deprecated = true]; + + // Произвольные данные в формате списка из пар ключ-значения + repeated Pair metadata_pairs = 5; +} + +// Ответ на публикацию набора сообщений +message PublishBatchResponse { + // Идентификаторы сообщений + repeated uint64 ids = 1; + // Содержит дополнительные данные необходимые для отладки и трассировки + map metadata = 2 [deprecated = true]; + // Флаги наличия дубликатов + repeated bool is_duplicates = 3; +} + +// Ответ на публикацию сообщения +message PublishResponse { + // Идентификатор сообщения добавленного в очередь + // (возможно не нужно) + uint64 id = 1; + // Содержит дополнительные данные необходимые для отладки и трассировки + map metadata = 2 [deprecated = true]; + // Если true, то был дубликат сообщения + bool is_duplicate = 3; +} + +// Зарос на публикацию сообщения через двусторонний стрим +message PublishStreamRequest { + // Идентификатор запроса на публикацию сообщения + uint64 request_id = 1; + + // Запрос на публикацию сообщения + PublishRequest request = 2; +} + +// Ответ на публикацию сообщения через двусторонний стрим +message PublishStreamResponse { + // Идентификатор запроса на публикацию сообщения + uint64 request_id = 1; + + oneof result { + // Сообщение об успешной публикации + PublishResponse success = 2; + // Сообщение об ошибке публикации + Error error = 3; + } +} + +// Запрос на публикацию набора сообщений через двусторонний стрим +message PublishBatchStreamRequest { + // Идентификатор запроса на публикацию сообщения + uint64 request_id = 1; + + // Запрос на публикацию набора сообщений + PublishBatchRequest request = 2; +} + +// Ответ на публикацию набора сообщений через двусторонний стрим +message PublishBatchStreamResponse { + // Идентификатор запроса на публикацию сообщения + uint64 request_id = 1; + + oneof result { + // Сообщение об успешной публикации + PublishBatchResponse success = 2; + // Сообщение об ошибке публикации + Error error = 3; + } +} + +// Запрос на рассылку сообщения на указанные шарды +message BroadcastRequest { + // Название очереди, в которую необходимо опубликовать сообщение + string queue = 1; + + // Ключ маршрутизации сообщения (тип сообщения) + // необходим для фильтрации сообщений из очереди на консьюмерах + optional string routing_key = 2; + + // Ключ дедупликации + // необходим для проверки повторных сообщений, + // если не указан, то проверка не производится + optional string deduplication_key = 3; + + // Произвольные данные в бинарном формате, содержит тело сообщения + bytes payload = 4; + + // Произвольные данные в бинарном формате, + // содержит дополнительные для сообщения данные, + // необходимые для отладки и трассировки + map metadata = 5 [deprecated = true]; + + // Список с названиями репликасетов, на которые нужно опубликовать сообщение. + // По умолчанию рассылка происходит на все шарды. + repeated string replicasets = 6; + + // Максимальное время на рассылку сообщения + optional uint64 timeout = 7; + + // Произвольные данные в формате списка из пар ключ-значения + repeated Pair metadata_pairs = 8; + + // Режим дедупликации сообщения + Deduplication deduplication = 9; +} + +// Сообщение об успешной публикации +message Success { + // Идентификатор сообщения добавленного в очередь + uint64 id = 1; + // Флаги наличия дубликатов + bool is_duplicate = 2; +} + +// Сообщение об ошибке публикации +message Error { + // Код ошибки + uint32 code = 1; + // Сообщение об ошибке + string message = 2; +} + +// Ответ репликасета на публикацию сообщения +message ReplicasetResponse { + // Сообщение с результатами публикации сообщения + oneof result { + // Сообщение об успешной публикации + Success success = 1; + + // Сообщение об ошибке публикации + Error error = 2; + } +} + +// Ответ на рассылку сообщения +message BroadcastResponse { + // Код завершения рассылки: + // 0 - Успешная публикация + // 1 - Ошибка на роутере + // 2 - Ошибка на репликасете + uint32 code = 1; + + // Сообщение об ошибке + optional string error = 2; + + // Набор ответов с шардов + map replicasets = 3; + + // Содержит дополнительные данные необходимые для отладки и трассировки + map metadata = 4 [deprecated = true]; +} diff --git a/testcontainers/src/test/proto/messages/cursor.proto b/testcontainers/src/test/proto/tqe3/messages/cursor.proto similarity index 100% rename from testcontainers/src/test/proto/messages/cursor.proto rename to testcontainers/src/test/proto/tqe3/messages/cursor.proto diff --git a/testcontainers/src/test/proto/messages/message.proto b/testcontainers/src/test/proto/tqe3/messages/message.proto similarity index 100% rename from testcontainers/src/test/proto/messages/message.proto rename to testcontainers/src/test/proto/tqe3/messages/message.proto diff --git a/testcontainers/src/test/proto/services/consumer.proto b/testcontainers/src/test/proto/tqe3/services/consumer.proto similarity index 100% rename from testcontainers/src/test/proto/services/consumer.proto rename to testcontainers/src/test/proto/tqe3/services/consumer.proto diff --git a/testcontainers/src/test/proto/services/producer.proto b/testcontainers/src/test/proto/tqe3/services/producer.proto similarity index 100% rename from testcontainers/src/test/proto/services/producer.proto rename to testcontainers/src/test/proto/tqe3/services/producer.proto diff --git a/testcontainers/src/test/resources/tqe2/simple-config/simple-grpc.yml b/testcontainers/src/test/resources/tqe2/simple-config/simple-grpc.yml new file mode 100644 index 00000000..d8044bac --- /dev/null +++ b/testcontainers/src/test/resources/tqe2/simple-config/simple-grpc.yml @@ -0,0 +1,21 @@ +core_port: 1111 +grpc_listen: + - uri: 'tcp://0.0.0.0:18182' + +publisher: + enabled: true + tarantool: + user: test-super + pass: test + connections: + routers: + - "router:3301" + +consumer: + enabled: true + tarantool: + user: test-super + pass: test + connections: + storage: + - "master:3301" diff --git a/testcontainers/src/test/resources/tqe2/simple-config/simple-queue.yml b/testcontainers/src/test/resources/tqe2/simple-config/simple-queue.yml new file mode 100644 index 00000000..63397f99 --- /dev/null +++ b/testcontainers/src/test/resources/tqe2/simple-config/simple-queue.yml @@ -0,0 +1,66 @@ +# Credentials +credentials: + users: + test-super: + password: 'test' + roles: [ super ] + admin: + password: 'secret-cluster-cookie' + roles: [ super ] + replicator: + password: 'secret' + roles: [ replication ] + storage: + roles: [ sharding ] + password: storage + +# advertise configs for all nodes +iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + password: storage + +roles: [ roles.metrics-export ] +# queues configs +roles_cfg: + app.roles.queue: + queues: + - name: test + deduplication_mode: keep_latest + disabled_filters_by: [ sharding_key ] + roles.metrics-export: + http: + - listen: 8081 + endpoints: + - format: prometheus + path: '/metrics' + +groups: + routers: + replicasets: + r-1: + sharding: + roles: [ router ] + roles: [ app.roles.api ] + instances: + router: + iproto: + listen: + - uri: router:3301 + storages: + replicasets: + shard-1: + replication: + failover: manual + sharding: + roles: [ storage ] + roles: [ app.roles.queue ] + leader: master + instances: + master: + iproto: + listen: + - uri: master:3301 diff --git a/testcontainers/src/test/resources/tqe/simple-config/simple-grpc.yml b/testcontainers/src/test/resources/tqe3/simple-config/simple-grpc.yml similarity index 84% rename from testcontainers/src/test/resources/tqe/simple-config/simple-grpc.yml rename to testcontainers/src/test/resources/tqe3/simple-config/simple-grpc.yml index 2c4c609f..9353580b 100644 --- a/testcontainers/src/test/resources/tqe/simple-config/simple-grpc.yml +++ b/testcontainers/src/test/resources/tqe3/simple-config/simple-grpc.yml @@ -1,6 +1,6 @@ core_port: 1111 grpc_listen: -- uri: 'tcp://0.0.0.0:18182' + - uri: 'tcp://0.0.0.0:18182' producer: enabled: true @@ -13,7 +13,7 @@ producer: retry_delay: 5s connections: routers: - - "router:3301" + - "router:3301" consumer: enabled: true @@ -26,4 +26,4 @@ consumer: retry_delay: 5s connections: storage: - - "master:3301" + - "master:3301" diff --git a/testcontainers/src/test/resources/tqe/simple-config/simple-queue.yml b/testcontainers/src/test/resources/tqe3/simple-config/simple-queue.yml similarity index 100% rename from testcontainers/src/test/resources/tqe/simple-config/simple-queue.yml rename to testcontainers/src/test/resources/tqe3/simple-config/simple-queue.yml From da4c8df821f53bda6cadc0e4dc74834dc6b770e4 Mon Sep 17 00:00:00 2001 From: Dmitry Kasimovskiy Date: Thu, 4 Jun 2026 10:51:02 +0300 Subject: [PATCH 2/5] refactor(testcontainers): reduce duplication in TQE2/TQE3 test classes Add shared helpers (loadConfig, createInvalidGrpcConfigStream, createInvalidConfigsPathsStream) to AbstractTQETest and simplify static config loading and data providers in all four test classes. Co-Authored-By: Claude Opus 4.7 --- .../integration/tqe/AbstractTQETest.java | 50 +++++++++++++++++++ .../tqe/FileTQE2ConfiguratorTest.java | 41 ++------------- .../tqe/FileTQE3ConfiguratorTest.java | 41 ++------------- .../integration/tqe/TQE2ClusterImplTest.java | 40 ++------------- .../integration/tqe/TQE3ClusterImplTest.java | 40 ++------------- 5 files changed, 62 insertions(+), 150 deletions(-) diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQETest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQETest.java index 2e3c9dfe..b18c643d 100644 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQETest.java +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQETest.java @@ -5,15 +5,24 @@ package org.testcontainers.containers.integration.tqe; +import java.io.IOException; import java.net.InetSocketAddress; +import java.net.URISyntaxException; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.provider.Arguments; import org.rnorth.ducttape.unreliables.Unreliables; import org.testcontainers.utility.DockerImageName; @@ -27,6 +36,16 @@ abstract class AbstractTQETest { protected abstract Path simpleQueueConfig(); + protected static Path loadConfig(String resourcePath) { + try { + return Paths.get( + Objects.requireNonNull(AbstractTQETest.class.getClassLoader().getResource(resourcePath)) + .toURI()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + protected static ManagedChannel createReadyChannel(InetSocketAddress address) { return Unreliables.retryUntilSuccess( 60, @@ -158,4 +177,35 @@ protected static List invalidGrpcConfigs(String producerRoleName) { """ .formatted(producerRoleName)); } + + protected static Stream createInvalidGrpcConfigStream( + String producerRoleName, Path tempDir) { + return invalidGrpcConfigs(producerRoleName).stream() + .map( + s -> { + final Path testConfigPath = tempDir.resolve(UUID.randomUUID() + ".yml"); + try { + Files.writeString(testConfigPath, s); + return testConfigPath; + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + protected static Stream createInvalidConfigsPathsStream( + Path simpleQueueConfig, Path simpleGrpcConfig, Path tempDir) { + return Stream.of( + // invalid grpc configs + // null + Arguments.of(simpleQueueConfig, null), + // empty + Arguments.of(simpleQueueConfig, Set.of()), + // non regular + Arguments.of(simpleQueueConfig, Set.of(tempDir)), + + // invalid queue config + Arguments.of(null, Set.of(simpleGrpcConfig)), + Arguments.of(tempDir, Set.of(simpleGrpcConfig))); + } } diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE2ConfiguratorTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE2ConfiguratorTest.java index f9ae15d3..ce3f9300 100644 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE2ConfiguratorTest.java +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE2ConfiguratorTest.java @@ -6,11 +6,8 @@ package org.testcontainers.containers.integration.tqe; import java.io.IOException; -import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.stream.Stream; @@ -33,29 +30,8 @@ class FileTQE2ConfiguratorTest extends AbstractTQETest { System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") + "tarantool/message-queue-ee:2.5.3"); - private static final Path SIMPLE_GRPC_CONFIG; - private static final Path SIMPLE_QUEUE_CONFIG; - - static { - try { - SIMPLE_GRPC_CONFIG = - Paths.get( - Objects.requireNonNull( - FileTQE2ConfiguratorTest.class - .getClassLoader() - .getResource("tqe2/simple-config/simple-grpc.yml")) - .toURI()); - SIMPLE_QUEUE_CONFIG = - Paths.get( - Objects.requireNonNull( - FileTQE2ConfiguratorTest.class - .getClassLoader() - .getResource("tqe2/simple-config/simple-queue.yml")) - .toURI()); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } + private static final Path SIMPLE_GRPC_CONFIG = loadConfig("tqe2/simple-config/simple-grpc.yml"); + private static final Path SIMPLE_QUEUE_CONFIG = loadConfig("tqe2/simple-config/simple-queue.yml"); @Override protected DockerImageName imageName() { @@ -235,18 +211,7 @@ void testInvalidQueueConfig(String invalidQueueConfig) throws IOException { } public static Stream dataForTestInvalidConfigsPaths() { - return Stream.of( - // invalid grpc configs - // null - Arguments.of(SIMPLE_QUEUE_CONFIG, null), - // empty - Arguments.of(SIMPLE_QUEUE_CONFIG, Set.of()), - // non regular - Arguments.of(SIMPLE_QUEUE_CONFIG, Set.of(TEST_TEMP_DIR)), - - // invalid queue config - Arguments.of(null, Set.of(SIMPLE_GRPC_CONFIG)), - Arguments.of(TEST_TEMP_DIR, Set.of(SIMPLE_GRPC_CONFIG))); + return createInvalidConfigsPathsStream(SIMPLE_QUEUE_CONFIG, SIMPLE_GRPC_CONFIG, TEST_TEMP_DIR); } @ParameterizedTest diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE3ConfiguratorTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE3ConfiguratorTest.java index 1a24870c..672a050e 100644 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE3ConfiguratorTest.java +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE3ConfiguratorTest.java @@ -6,11 +6,8 @@ package org.testcontainers.containers.integration.tqe; import java.io.IOException; -import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.stream.Stream; @@ -33,29 +30,8 @@ class FileTQE3ConfiguratorTest extends AbstractTQETest { System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") + "tarantool/message-queue-ee:v3.5.0"); - private static final Path SIMPLE_GRPC_CONFIG; - private static final Path SIMPLE_QUEUE_CONFIG; - - static { - try { - SIMPLE_GRPC_CONFIG = - Paths.get( - Objects.requireNonNull( - FileTQE3ConfiguratorTest.class - .getClassLoader() - .getResource("tqe3/simple-config/simple-grpc.yml")) - .toURI()); - SIMPLE_QUEUE_CONFIG = - Paths.get( - Objects.requireNonNull( - FileTQE3ConfiguratorTest.class - .getClassLoader() - .getResource("tqe3/simple-config/simple-queue.yml")) - .toURI()); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } + private static final Path SIMPLE_GRPC_CONFIG = loadConfig("tqe3/simple-config/simple-grpc.yml"); + private static final Path SIMPLE_QUEUE_CONFIG = loadConfig("tqe3/simple-config/simple-queue.yml"); @Override protected DockerImageName imageName() { @@ -239,18 +215,7 @@ void testInvalidQueueConfig(String invalidQueueConfig) throws IOException { } public static Stream dataForTestInvalidConfigsPaths() { - return Stream.of( - // invalid grpc configs - // null - Arguments.of(SIMPLE_QUEUE_CONFIG, null), - // empty - Arguments.of(SIMPLE_QUEUE_CONFIG, Set.of()), - // non regular - Arguments.of(SIMPLE_QUEUE_CONFIG, Set.of(TEST_TEMP_DIR)), - - // invalid queue config - Arguments.of(null, Set.of(SIMPLE_GRPC_CONFIG)), - Arguments.of(TEST_TEMP_DIR, Set.of(SIMPLE_GRPC_CONFIG))); + return createInvalidConfigsPathsStream(SIMPLE_QUEUE_CONFIG, SIMPLE_GRPC_CONFIG, TEST_TEMP_DIR); } @ParameterizedTest diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java index 8017995c..c1823968 100644 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java @@ -7,15 +7,12 @@ import java.io.IOException; import java.net.InetSocketAddress; -import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.time.Duration; import java.util.Arrays; import java.util.LinkedHashSet; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -64,29 +61,8 @@ class TQE2ClusterImplTest extends AbstractTQETest { System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") + "tarantool/message-queue-ee:2.5.3"); - private static final Path SIMPLE_GRPC_CONFIG; - private static final Path SIMPLE_QUEUE_CONFIG; - - static { - try { - SIMPLE_GRPC_CONFIG = - Paths.get( - Objects.requireNonNull( - TQE2ClusterImplTest.class - .getClassLoader() - .getResource("tqe2/simple-config/simple-grpc.yml")) - .toURI()); - SIMPLE_QUEUE_CONFIG = - Paths.get( - Objects.requireNonNull( - TQE2ClusterImplTest.class - .getClassLoader() - .getResource("tqe2/simple-config/simple-queue.yml")) - .toURI()); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } + private static final Path SIMPLE_GRPC_CONFIG = loadConfig("tqe2/simple-config/simple-grpc.yml"); + private static final Path SIMPLE_QUEUE_CONFIG = loadConfig("tqe2/simple-config/simple-queue.yml"); @Override protected DockerImageName imageName() { @@ -282,17 +258,7 @@ void testInvalidQueueConfigShouldThrow(Path queueConfig) { } public static Stream dataForTestInvalidGrpcConfig() { - return invalidGrpcConfigs("publisher").stream() - .map( - s -> { - final Path testConfigPath = TEST_TEMP_DIR.resolve(UUID.randomUUID() + ".yml"); - try { - Files.writeString(testConfigPath, s); - return testConfigPath; - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + return createInvalidGrpcConfigStream("publisher", TEST_TEMP_DIR); } @ParameterizedTest diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java index 9f8985d9..6d66b3aa 100644 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java @@ -7,15 +7,12 @@ import java.io.IOException; import java.net.InetSocketAddress; -import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.time.Duration; import java.util.Arrays; import java.util.LinkedHashSet; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -65,29 +62,8 @@ class TQE3ClusterImplTest extends AbstractTQETest { System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") + "tarantool/message-queue-ee:v3.5.0"); - private static final Path SIMPLE_GRPC_CONFIG; - private static final Path SIMPLE_QUEUE_CONFIG; - - static { - try { - SIMPLE_GRPC_CONFIG = - Paths.get( - Objects.requireNonNull( - TQE3ClusterImplTest.class - .getClassLoader() - .getResource("tqe3/simple-config/simple-grpc.yml")) - .toURI()); - SIMPLE_QUEUE_CONFIG = - Paths.get( - Objects.requireNonNull( - TQE3ClusterImplTest.class - .getClassLoader() - .getResource("tqe3/simple-config/simple-queue.yml")) - .toURI()); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } + private static final Path SIMPLE_GRPC_CONFIG = loadConfig("tqe3/simple-config/simple-grpc.yml"); + private static final Path SIMPLE_QUEUE_CONFIG = loadConfig("tqe3/simple-config/simple-queue.yml"); @Override protected DockerImageName imageName() { @@ -287,17 +263,7 @@ void testInvalidQueueConfigShouldThrow(Path queueConfig) { } public static Stream dataForTestInvalidGrpcConfig() { - return invalidGrpcConfigs("producer").stream() - .map( - s -> { - final Path testConfigPath = TEST_TEMP_DIR.resolve(UUID.randomUUID() + ".yml"); - try { - Files.writeString(testConfigPath, s); - return testConfigPath; - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + return createInvalidGrpcConfigStream("producer", TEST_TEMP_DIR); } @ParameterizedTest From 4d5c877c7c16d94302318ebfc3487fc8e7faea38 Mon Sep 17 00:00:00 2001 From: Dmitry Kasimovskiy Date: Thu, 4 Jun 2026 16:02:52 +0300 Subject: [PATCH 3/5] refactor(testcontainers): extract common cluster tests into AbstractTQEClusterTest Move shared cluster tests (testMultiplyRestart, testRestartMethod, testInvalidQueueConfigShouldThrow, testInvalidGrpcConfig) into a new AbstractTQEClusterTest base class. TQE2ClusterImplTest and TQE3ClusterImplTest now extend it, eliminating duplication. Also revert FileTQEConfigurator to eager initialization (fields back to final) and extract inline YAML configs to resource files. Co-Authored-By: Claude Opus 4.7 --- .../configuration/FileTQEConfigurator.java | 104 +++---- .../tqe/AbstractTQEClusterTest.java | 87 ++++++ .../integration/tqe/AbstractTQETest.java | 59 ++++ .../tqe/FileTQE2ConfiguratorTest.java | 146 +-------- .../tqe/FileTQE3ConfiguratorTest.java | 148 +-------- .../integration/tqe/TQE2ClusterImplTest.java | 288 ++--------------- .../integration/tqe/TQE3ClusterImplTest.java | 292 ++---------------- .../invalid-config/no-consumer-storage.yml | 52 ++++ .../invalid-config/no-test-super-user.yml | 63 ++++ .../router-no-required-roles.yml | 61 ++++ .../invalid-config/storage-no-listen-uri.yml | 61 ++++ .../invalid-config/no-consumer-storage.yml | 53 ++++ .../invalid-config/no-test-super-user.yml | 66 ++++ .../router-no-required-roles.yml | 63 ++++ .../invalid-config/storage-no-listen-uri.yml | 62 ++++ 15 files changed, 759 insertions(+), 846 deletions(-) create mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQEClusterTest.java create mode 100644 testcontainers/src/test/resources/tqe2/invalid-config/no-consumer-storage.yml create mode 100644 testcontainers/src/test/resources/tqe2/invalid-config/no-test-super-user.yml create mode 100644 testcontainers/src/test/resources/tqe2/invalid-config/router-no-required-roles.yml create mode 100644 testcontainers/src/test/resources/tqe2/invalid-config/storage-no-listen-uri.yml create mode 100644 testcontainers/src/test/resources/tqe3/invalid-config/no-consumer-storage.yml create mode 100644 testcontainers/src/test/resources/tqe3/invalid-config/no-test-super-user.yml create mode 100644 testcontainers/src/test/resources/tqe3/invalid-config/router-no-required-roles.yml create mode 100644 testcontainers/src/test/resources/tqe3/invalid-config/storage-no-listen-uri.yml diff --git a/testcontainers/src/main/java/org/testcontainers/containers/tqe/configuration/FileTQEConfigurator.java b/testcontainers/src/main/java/org/testcontainers/containers/tqe/configuration/FileTQEConfigurator.java index 8af8e73c..d6dcd27c 100644 --- a/testcontainers/src/main/java/org/testcontainers/containers/tqe/configuration/FileTQEConfigurator.java +++ b/testcontainers/src/main/java/org/testcontainers/containers/tqe/configuration/FileTQEConfigurator.java @@ -88,15 +88,15 @@ public class FileTQEConfigurator implements TQEConfigurator { private final Duration bootstrapTimeout; - private Map> queue; + private final Map> queue; - private Map> grpc; + private final Map> grpc; private final Set routerNames; - private Tarantool3Configuration queueParsedConfig; + private final Tarantool3Configuration queueParsedConfig; - private Network network; + private final Network network; private final String routerRole; @@ -118,65 +118,71 @@ private FileTQEConfigurator( this.routerNames = new LinkedHashSet<>(1); this.image = image; this.startupTimeout = startupTimeout; + this.network = Network.newNetwork(); + this.queueParsedConfig = ConfigurationUtils.readFromFile(this.queueConfig); + this.queue = initQueue(this.network, this.image, this.clusterName, this.startupTimeout); + this.grpc = + initGrpc( + this.grpcConfigs, + this.network, + this.image, + this.clusterName, + this.queue, + this.startupTimeout); } - private synchronized Tarantool3Configuration parsedConfig() { - if (this.queueParsedConfig == null) { - this.queueParsedConfig = ConfigurationUtils.readFromFile(this.queueConfig); - } - return this.queueParsedConfig; - } - - private synchronized Network setupNetwork() { - if (this.network == null) { - this.network = Network.newNetwork(); - } - return this.network; - } - - private Map> createGrpcContainers() { + private Map> initGrpc( + Collection grpcConfigs, + Network network, + DockerImageName image, + String clusterName, + Map> queue, + Duration startupTimeout) { Map> grpc = new LinkedHashMap<>(); int i = 1; - for (Path grpcConfig : this.grpcConfigs) { + for (Path grpcConfig : grpcConfigs) { final String grpcName = "grpc-" + i; - final String containerName = grpcName + "-" + this.clusterName; + final String containerName = grpcName + "-" + clusterName; GrpcContainerImpl container = - new GrpcContainerImpl(this.image, grpcConfig, grpcName, this.startupTimeout) - .withNetwork(setupNetwork()) + new GrpcContainerImpl(image, grpcConfig, grpcName, startupTimeout) + .withNetwork(network) .withCreateContainerCmdModifier(cmd -> cmd.withName(containerName)) .withNetworkAliases(containerName) - .dependsOn(queue().values()); + .dependsOn(queue.values()); grpc.put(grpcName, container); } return grpc; } - private Map> createQueueContainers() { + private Map> initQueue( + Network network, DockerImageName image, String clusterName, Duration startupTimeout) { - final List instances = ConfigurationUtils.parseInstances(parsedConfig()); + final List instances = ConfigurationUtils.parseInstances(this.queueParsedConfig); final Map> nodes = new LinkedHashMap<>(instances.size()); this.routerNames.addAll( - ConfigurationUtils.findInstancesWithRole(parsedConfig(), this.routerRole)); + ConfigurationUtils.findInstancesWithRole(this.queueParsedConfig, this.routerRole)); if (this.routerNames.isEmpty()) { LOGGER.error("At least one container must have the 'router' and '{}' roles", this.routerRole); throw new ContainerLaunchException(CONFIGURATOR_ERROR_MSG); } - final Map advertiseClientUris = resolveAdvertiseClientUris(parsedConfig()); - final Map advertisePeerUris = resolveAdvertisePeerUris(parsedConfig()); - final Map> listenUri = resolveListenUris(parsedConfig()); + final Map advertiseClientUris = + resolveAdvertiseClientUris(this.queueParsedConfig); + final Map advertisePeerUris = + resolveAdvertisePeerUris(this.queueParsedConfig); + final Map> listenUri = resolveListenUris(this.queueParsedConfig); if (listenUri.size() != instances.size()) { LOGGER.error("All nodes should have 'listen.uri' parameter!"); throw new ContainerLaunchException(CONFIGURATOR_ERROR_MSG); } for (String instance : instances) { - final String containerName = instance + "-" + this.clusterName; + final String containerName = instance + "-" + clusterName; final Tarantool3Container container = - new Tarantool3Container(this.image, instance) - .withNetwork(setupNetwork()) - .withConfigPath(this.queueConfig) + new Tarantool3Container(image, instance) + .withNetwork(network) + .withConfigPath(queueConfig) .withNetworkAliases(containerName) .withCreateContainerCmdModifier(cmd -> cmd.withUser("root").withName(containerName)) .waitingFor( @@ -184,7 +190,7 @@ private Map> createQueueContainers() { instance, TCMConfig.DEFAULT_TARANTOOL_USERNAME, TCMConfig.DEFAULT_TARANTOOL_PASSWORD)) - .withStartupTimeout(this.startupTimeout) + .withStartupTimeout(startupTimeout) .withCommand("tarantool"); final HostPort advertiseClientUri = advertiseClientUris.get(instance); @@ -377,25 +383,19 @@ public String clusterName() { } @Override - public synchronized Map> queue() { - if (this.queue == null) { - this.queue = createQueueContainers(); - } + public Map> queue() { return this.queue; } @Override - public synchronized Map> grpc() { - if (this.grpc == null) { - this.grpc = createGrpcContainers(); - } + public Map> grpc() { return this.grpc; } @Override public synchronized void configure() { final Entry> router = - queue().entrySet().stream() + this.queue.entrySet().stream() .filter(e -> this.routerNames.contains(e.getKey())) .findFirst() .orElseThrow( @@ -405,13 +405,13 @@ public synchronized void configure() { }); final HostPort hostForRouter = - resolveAdvertiseClientUris(parsedConfig()).entrySet().stream() + resolveAdvertiseClientUris(this.queueParsedConfig).entrySet().stream() .filter(e -> Objects.equals(e.getKey(), router.getKey())) .map(Entry::getValue) .findFirst() .orElseGet( () -> - resolveListenUris(parsedConfig()).entrySet().stream() + resolveListenUris(this.queueParsedConfig).entrySet().stream() .filter(e -> Objects.equals(e.getKey(), router.getKey())) .filter(e -> !e.getValue().isEmpty()) .map(e -> e.getValue().get(0)) @@ -434,15 +434,9 @@ public synchronized boolean isConfigured() { @Override public synchronized void close() { - if (this.queue != null) { - this.queue.values().parallelStream().forEach(Startable::close); - } - if (this.grpc != null) { - this.grpc.values().parallelStream().forEach(Startable::close); - } - if (this.network != null) { - this.network.close(); - } + this.queue.values().parallelStream().forEach(Startable::close); + this.grpc.values().parallelStream().forEach(Startable::close); + this.network.close(); } /** diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQEClusterTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQEClusterTest.java new file mode 100644 index 00000000..8d09f292 --- /dev/null +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQEClusterTest.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY + * All Rights Reserved. + */ + +package org.testcontainers.containers.integration.tqe; + +import java.nio.file.Path; +import java.time.Duration; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.RepeatedTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.testcontainers.containers.ContainerLaunchException; +import org.testcontainers.containers.tqe.TQECluster; +import org.testcontainers.containers.tqe.configuration.TQEConfigurator; + +abstract class AbstractTQEClusterTest extends AbstractTQETest { + + protected abstract TQECluster createCluster(TQEConfigurator configurator); + + protected abstract TQEConfigurator createConfigurator(Path queueConfig, Set grpcConfigs); + + protected abstract TQEConfigurator createConfiguratorWithTimeout( + Path queueConfig, Set grpcConfigs, Duration startupTimeout); + + // --- Common cluster tests --- + + @RepeatedTest(3) + void testMultiplyRestart() throws Exception { + try (TQEConfigurator configurator = + createConfigurator(simpleQueueConfig(), Set.of(simpleGrpcConfig())); + TQECluster cluster = createCluster(configurator)) { + cluster.start(); + } + } + + @Test + void testRestartMethod() throws Exception { + try (TQEConfigurator configurator = + createConfigurator(simpleQueueConfig(), Set.of(simpleGrpcConfig())); + TQECluster cluster = createCluster(configurator)) { + cluster.start(); + cluster.restart(1, TimeUnit.SECONDS, 1, TimeUnit.SECONDS); + } + } + + public abstract Stream dataForTestInvalidQueueConfigShouldThrow(); + + @ParameterizedTest + @MethodSource("dataForTestInvalidQueueConfigShouldThrow") + void testInvalidQueueConfigShouldThrow(Path queueConfig) { + Assertions.assertThrows( + ContainerLaunchException.class, + () -> { + try (TQEConfigurator configurator = + createConfiguratorWithTimeout( + queueConfig, Set.of(simpleGrpcConfig()), Duration.ofSeconds(5)); + TQECluster cluster = createCluster(configurator)) { + cluster.start(); + } + }); + } + + public abstract Stream dataForTestInvalidGrpcConfig(); + + @ParameterizedTest + @MethodSource("dataForTestInvalidGrpcConfig") + void testInvalidGrpcConfig(Path grpcConfig) { + Assertions.assertThrows( + ContainerLaunchException.class, + () -> { + try (TQEConfigurator configurator = + createConfiguratorWithTimeout( + simpleQueueConfig(), Set.of(grpcConfig), Duration.ofSeconds(5)); + TQECluster cluster = createCluster(configurator)) { + cluster.start(); + } + }); + } +} diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQETest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQETest.java index b18c643d..2caf3881 100644 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQETest.java +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQETest.java @@ -12,24 +12,38 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; +import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.protobuf.ByteString; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; +import org.instancio.Instancio; +import org.instancio.Select; +import org.instancio.generators.Generators; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.provider.Arguments; import org.rnorth.ducttape.unreliables.Unreliables; +import org.testcontainers.containers.tqe.GrpcContainer; +import org.testcontainers.containers.tqe.GrpcContainer.GrpcRole; +import org.testcontainers.containers.tqe.TQECluster; +import org.testcontainers.containers.utils.pojo.User; import org.testcontainers.utility.DockerImageName; abstract class AbstractTQETest { @TempDir protected static Path TEST_TEMP_DIR; + protected static final ObjectMapper MAPPER = new ObjectMapper(); + protected abstract DockerImageName imageName(); protected abstract Path simpleGrpcConfig(); @@ -46,6 +60,10 @@ protected static Path loadConfig(String resourcePath) { } } + protected static List loadConfigs(String... resourcePaths) { + return Arrays.stream(resourcePaths).map(AbstractTQETest::loadConfig).toList(); + } + protected static ManagedChannel createReadyChannel(InetSocketAddress address) { return Unreliables.retryUntilSuccess( 60, @@ -208,4 +226,45 @@ protected static Stream createInvalidConfigsPathsStream( Arguments.of(null, Set.of(simpleGrpcConfig)), Arguments.of(tempDir, Set.of(simpleGrpcConfig))); } + + protected static List generateTestUsers() { + return Instancio.ofList(User.class) + .size(100) + .generate( + Select.field(User::getName), g -> g.string().alphaNumeric().allowEmpty().nullable()) + .generate(Select.field(User::getAge), Generators::ints) + .create(); + } + + protected static InetSocketAddress findGrpcAddress(TQECluster cluster, GrpcRole role) { + final List> containers = + cluster.grpc().values().stream().filter(g -> g.roles().contains(role)).toList(); + Assertions.assertFalse(containers.isEmpty(), "No gRPC container with role: " + role); + final Optional address = + containers.get(0).grpcAddresses().stream().findFirst(); + Assertions.assertTrue(address.isPresent(), "No gRPC address for role: " + role); + return address.get(); + } + + protected static User deserializeUser(ByteString payload) { + try { + return MAPPER.readValue(payload.toByteArray(), User.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + protected static ByteString serializeUser(User user) { + try { + return ByteString.copyFrom(MAPPER.writeValueAsBytes(user)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + protected static void assertAllUsersConsumed(List users, Set result) { + Unreliables.retryUntilTrue( + 60, TimeUnit.SECONDS, () -> new LinkedHashSet<>(users).size() == result.size()); + Assertions.assertEquals(new LinkedHashSet<>(users), result); + } } diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE2ConfiguratorTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE2ConfiguratorTest.java index ce3f9300..8d39f2c1 100644 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE2ConfiguratorTest.java +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE2ConfiguratorTest.java @@ -5,11 +5,8 @@ package org.testcontainers.containers.integration.tqe; -import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Set; -import java.util.UUID; import java.util.stream.Stream; import org.junit.jupiter.api.Assertions; @@ -55,155 +52,30 @@ void simpleConfiguration() { imageName(), simpleQueueConfig(), Set.of(simpleGrpcConfig())) .build()) { configurator.queue().values().parallelStream().forEach(Startable::start); + // TQE 2.x does not require explicit configure() — the router role bootstraps automatically configurator.grpc().values().parallelStream().forEach(Startable::start); } catch (Exception e) { throw new RuntimeException(e); } } - public static Stream dataForTestInvalidQueueConfigShouldThrow() { - return Stream.of( - // router have no required roles - """ - # Credentials - credentials: - users: - test-super: - password: 'test' - roles: [ super ] - admin: - password: 'secret-cluster-cookie' - roles: [ super ] - replicator: - password: 'secret' - roles: [ replication ] - storage: - roles: [ sharding ] - password: storage - # advertise configs for all nodes - iproto: - advertise: - peer: - login: replicator - sharding: - login: storage - password: storage - roles: [ roles.metrics-export ] - # queues configs - roles_cfg: - app.roles.queue: - queues: - - name: test - deduplication_mode: keep_latest - disabled_filters_by: [ sharding_key ] - roles.metrics-export: - http: - - listen: 8081 - endpoints: - - format: prometheus - path: '/metrics' - groups: - routers: - replicasets: - r-1: - sharding: - roles: [ router ] - instances: - router: - iproto: - listen: - - uri: router:3301 - storages: - replicasets: - shard-1: - replication: - failover: manual - sharding: - roles: [ storage ] - leader: master - instances: - master: - iproto: - listen: - - uri: master:3301 - """, - """ - # Credentials - credentials: - users: - test-super: - password: 'test' - roles: [ super ] - admin: - password: 'secret-cluster-cookie' - roles: [ super ] - replicator: - password: 'secret' - roles: [ replication ] - storage: - roles: [ sharding ] - password: storage - # advertise configs for all nodes - iproto: - advertise: - peer: - login: replicator - sharding: - login: storage - password: storage - roles: [ roles.metrics-export ] - # queues configs - roles_cfg: - app.roles.queue: - queues: - - name: test - deduplication_mode: keep_latest - disabled_filters_by: [ sharding_key ] - roles.metrics-export: - http: - - listen: 8081 - endpoints: - - format: prometheus - path: '/metrics' - groups: - routers: - replicasets: - r-1: - sharding: - roles: [ router ] - roles: [ app.roles.api ] - instances: - router: - iproto: - listen: - - uri: router:3301 - storages: - replicasets: - shard-1: - replication: - failover: manual - sharding: - roles: [ storage ] - leader: master - instances: - master: - iproto: - net_msg_max: 768 - """); + public static Stream dataForTestInvalidQueueConfigShouldThrow() { + return loadConfigs( + "tqe2/invalid-config/router-no-required-roles.yml", + "tqe2/invalid-config/storage-no-listen-uri.yml") + .stream() + .map(Arguments::of); } @ParameterizedTest @MethodSource("dataForTestInvalidQueueConfigShouldThrow") - void testInvalidQueueConfig(String invalidQueueConfig) throws IOException { - final Path invalidConfigPath = TEST_TEMP_DIR.resolve(UUID.randomUUID().toString()); - Files.writeString(invalidConfigPath, invalidQueueConfig); - + void testInvalidQueueConfig(Path invalidQueueConfig) { Assertions.assertThrows( ContainerLaunchException.class, () -> { try (FileTQEConfigurator c = FileTQEConfigurator.tqe2Builder( - imageName(), invalidConfigPath, Set.of(simpleGrpcConfig())) + imageName(), invalidQueueConfig, Set.of(simpleGrpcConfig())) .build()) { c.queue(); } diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE3ConfiguratorTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE3ConfiguratorTest.java index 672a050e..02cc932e 100644 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE3ConfiguratorTest.java +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE3ConfiguratorTest.java @@ -5,11 +5,8 @@ package org.testcontainers.containers.integration.tqe; -import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Set; -import java.util.UUID; import java.util.stream.Stream; import org.junit.jupiter.api.Assertions; @@ -62,152 +59,23 @@ void simpleConfiguration() { } } - public static Stream dataForTestInvalidQueueConfigShouldThrow() { - return Stream.of( - // router have no required roles - """ - # Credentials - credentials: - users: - test-super: - password: 'test' - roles: [ super ] - admin: - password: 'secret-cluster-cookie' - roles: [ super ] - replicator: - password: 'secret' - roles: [ replication ] - storage: - roles: [ sharding ] - password: storage - # advertise configs for all nodes - iproto: - advertise: - peer: - login: replicator - sharding: - login: storage - password: storage - roles: [ roles.metrics-export ] - # queues configs - roles_cfg: - roles.tqe-storage: - queues: - - name: test - deduplication_mode: keep_latest - disabled_filters_by: [ sharding_key ] - roles.metrics-export: - http: - - listen: 8081 - endpoints: - - format: prometheus - path: '/metrics' - groups: - routers: - replicasets: - router-1: - sharding: - roles: [ router ] - instances: - router: - iproto: - listen: - - uri: router:3301 - storages: - replicasets: - storage-1: - replication: - failover: manual - sharding: - roles: [ storage ] - roles: - - roles.tqe-storage - leader: master - instances: - master: - iproto: - listen: - - uri: master:3301 - """, - """ - # Credentials - credentials: - users: - test-super: - password: 'test' - roles: [ super ] - admin: - password: 'secret-cluster-cookie' - roles: [ super ] - replicator: - password: 'secret' - roles: [ replication ] - storage: - roles: [ sharding ] - password: storage - # advertise configs for all nodes - iproto: - advertise: - peer: - login: replicator - sharding: - login: storage - password: storage - roles: [ roles.metrics-export ] - # queues configs - roles_cfg: - roles.tqe-storage: - queues: - - name: test - deduplication_mode: keep_latest - disabled_filters_by: [ sharding_key ] - roles.metrics-export: - http: - - listen: 8081 - endpoints: - - format: prometheus - path: '/metrics' - groups: - routers: - replicasets: - router-1: - sharding: - roles: [ router ] - roles: - - roles.tqe-router - instances: - router: - iproto: - listen: - - uri: router:3301 - storages: - replicasets: - storage-1: - replication: - failover: manual - sharding: - roles: [ storage ] - leader: master - instances: - master: - iproto: - net_msg_max: 768 - """); + public static Stream dataForTestInvalidQueueConfigShouldThrow() { + return loadConfigs( + "tqe3/invalid-config/router-no-required-roles.yml", + "tqe3/invalid-config/storage-no-listen-uri.yml") + .stream() + .map(Arguments::of); } @ParameterizedTest @MethodSource("dataForTestInvalidQueueConfigShouldThrow") - void testInvalidQueueConfig(String invalidQueueConfig) throws IOException { - final Path invalidConfigPath = TEST_TEMP_DIR.resolve(UUID.randomUUID().toString()); - Files.writeString(invalidConfigPath, invalidQueueConfig); - + void testInvalidQueueConfig(Path invalidQueueConfig) { Assertions.assertThrows( ContainerLaunchException.class, () -> { try (FileTQEConfigurator c = FileTQEConfigurator.tqe3Builder( - imageName(), invalidConfigPath, Set.of(simpleGrpcConfig())) + imageName(), invalidQueueConfig, Set.of(simpleGrpcConfig())) .build()) { c.queue(); } diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java index c1823968..ec0b8e1b 100644 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java @@ -5,43 +5,27 @@ package org.testcontainers.containers.integration.tqe; -import java.io.IOException; import java.net.InetSocketAddress; -import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; -import java.util.Arrays; -import java.util.LinkedHashSet; import java.util.List; -import java.util.Optional; import java.util.Set; -import java.util.UUID; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; -import com.google.protobuf.ByteString; import io.grpc.ManagedChannel; import io.grpc.stub.StreamObserver; -import org.instancio.Instancio; -import org.instancio.Select; -import org.instancio.generators.Generators; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.RepeatedTest; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; import org.rnorth.ducttape.unreliables.Unreliables; -import org.testcontainers.containers.ContainerLaunchException; -import org.testcontainers.containers.tqe.GrpcContainer; import org.testcontainers.containers.tqe.GrpcContainer.GrpcRole; import org.testcontainers.containers.tqe.TQE2ClusterImpl; import org.testcontainers.containers.tqe.TQECluster; import org.testcontainers.containers.tqe.configuration.FileTQEConfigurator; import org.testcontainers.containers.tqe.configuration.TQEConfigurator; import org.testcontainers.containers.utils.pojo.User; -import org.testcontainers.shaded.com.fasterxml.jackson.databind.ObjectMapper; import org.testcontainers.utility.DockerImageName; import tarantool.queue_ee.v2.Consumer.SubscriptionNotifications; import tarantool.queue_ee.v2.Consumer.SubscriptionRequest; @@ -52,9 +36,7 @@ import tarantool.queue_ee.v2.PublisherServiceGrpc; import tarantool.queue_ee.v2.PublisherServiceGrpc.PublisherServiceBlockingStub; -class TQE2ClusterImplTest extends AbstractTQETest { - - private static final ObjectMapper MAPPER = new ObjectMapper(); +class TQE2ClusterImplTest extends AbstractTQEClusterTest { private static final DockerImageName IMAGE_NAME = DockerImageName.parse( @@ -79,206 +61,41 @@ protected Path simpleQueueConfig() { return SIMPLE_QUEUE_CONFIG; } - @RepeatedTest(10) - void testMultiplyRestart() throws Exception { - try (TQEConfigurator configurator = - FileTQEConfigurator.tqe2Builder( - IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(SIMPLE_GRPC_CONFIG)) - .build(); - TQECluster cluster = new TQE2ClusterImpl(configurator)) { - cluster.start(); - } + @Override + protected TQECluster createCluster(TQEConfigurator configurator) { + return new TQE2ClusterImpl(configurator); } - @Test - void testRestartMethod() throws Exception { - try (TQEConfigurator configurator = - FileTQEConfigurator.tqe2Builder( - IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(SIMPLE_GRPC_CONFIG)) - .build(); - TQECluster cluster = new TQE2ClusterImpl(configurator)) { - cluster.start(); - cluster.restart(1, TimeUnit.SECONDS, 1, TimeUnit.SECONDS); - } + @Override + protected TQEConfigurator createConfigurator(Path queueConfig, Set grpcConfigs) { + return FileTQEConfigurator.tqe2Builder(IMAGE_NAME, queueConfig, grpcConfigs).build(); } - public static Stream dataForTestInvalidQueueConfigShouldThrow() { - final List invalidConfigs = - Arrays.asList( - // no required test-super user - """ - # Credentials - credentials: - users: - admin: - password: 'secret-cluster-cookie' - roles: [ super ] - replicator: - password: 'secret' - roles: [ replication ] - storage: - roles: [ sharding ] - password: storage - - # advertise configs for all nodes - iproto: - advertise: - peer: - login: replicator - sharding: - login: storage - password: storage - - roles: [ roles.metrics-export ] - # queues configs - roles_cfg: - app.roles.queue: - queues: - - name: test - deduplication_mode: keep_latest - disabled_filters_by: [ sharding_key ] - roles.metrics-export: - http: - - listen: 8081 - endpoints: - - format: prometheus - path: '/metrics' - - groups: - routers: - replicasets: - r-1: - sharding: - roles: [ router ] - roles: [ app.roles.api ] - instances: - router: - iproto: - listen: - - uri: router:3301 - storages: - replicasets: - shard-1: - replication: - failover: manual - sharding: - roles: [ storage ] - leader: master - instances: - master: - iproto: - listen: - - uri: master:3301 - net_msg_max: 768 - """, - // no consumer storage to connect from grpc - """ - # Credentials - credentials: - users: - test-super: - password: 'test' - roles: [ super ] - admin: - password: 'secret-cluster-cookie' - roles: [ super ] - replicator: - password: 'secret' - roles: [ replication ] - storage: - roles: [ sharding ] - password: storage - - # advertise configs for all nodes - iproto: - advertise: - peer: - login: replicator - sharding: - login: storage - password: storage - - roles: [ roles.metrics-export ] - # queues configs - roles_cfg: - app.roles.queue: - queues: - - name: test - deduplication_mode: keep_latest - disabled_filters_by: [ sharding_key ] - roles.metrics-export: - http: - - listen: 8081 - endpoints: - - format: prometheus - path: '/metrics' - - groups: - routers: - replicasets: - r-1: - sharding: - roles: [ router ] - roles: [ app.roles.api ] - instances: - router: - iproto: - listen: - - uri: router:3301 - """); - - return invalidConfigs.stream() - .map( - s -> { - final Path testConfigPath = TEST_TEMP_DIR.resolve(UUID.randomUUID().toString()); - try { - Files.writeString(testConfigPath, s); - return Arguments.of(testConfigPath); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + @Override + protected TQEConfigurator createConfiguratorWithTimeout( + Path queueConfig, Set grpcConfigs, Duration startupTimeout) { + return FileTQEConfigurator.tqe2Builder(IMAGE_NAME, queueConfig, grpcConfigs) + .withStartupTimeout(startupTimeout) + .build(); } - @ParameterizedTest - @MethodSource("dataForTestInvalidQueueConfigShouldThrow") - void testInvalidQueueConfigShouldThrow(Path queueConfig) { - Assertions.assertThrows( - ContainerLaunchException.class, - () -> { - try (TQEConfigurator configurator = - FileTQEConfigurator.tqe2Builder( - IMAGE_NAME, queueConfig, Set.of(SIMPLE_GRPC_CONFIG)) - .withStartupTimeout(Duration.ofSeconds(5)) - .build(); - TQECluster cluster = new TQE2ClusterImpl(configurator)) { - cluster.start(); - } - }); + @Override + public Stream dataForTestInvalidQueueConfigShouldThrow() { + return loadConfigs( + "tqe2/invalid-config/no-test-super-user.yml", + "tqe2/invalid-config/no-consumer-storage.yml") + .stream() + .map(Arguments::of); } - public static Stream dataForTestInvalidGrpcConfig() { + @Override + public Stream dataForTestInvalidGrpcConfig() { return createInvalidGrpcConfigStream("publisher", TEST_TEMP_DIR); } - @ParameterizedTest - @MethodSource("dataForTestInvalidGrpcConfig") - void testInvalidGrpcConfig(Path grpcConfig) { - Assertions.assertThrows( - ContainerLaunchException.class, - () -> { - try (TQEConfigurator configurator = - FileTQEConfigurator.tqe2Builder( - IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(grpcConfig)) - .withStartupTimeout(Duration.ofSeconds(5)) - .build(); - TQECluster cluster = new TQE2ClusterImpl(configurator)) { - cluster.start(); - } - }); - } + // --- TQE2-specific test --- - @RepeatedTest(10) + @RepeatedTest(3) void testPublishAndConsumeData() { Assertions.assertDoesNotThrow( () -> { @@ -291,30 +108,11 @@ void testPublishAndConsumeData() { final String queueName = "test"; - final List> publishers = - cluster.grpc().values().stream() - .filter(g -> g.roles().contains(GrpcRole.PUBLISHER)) - .toList(); - final List> consumers = - cluster.grpc().values().stream() - .filter(g -> g.roles().contains(GrpcRole.CONSUMER)) - .toList(); - - Assertions.assertFalse(publishers.isEmpty()); - Assertions.assertFalse(consumers.isEmpty()); - - final Set grpcAddresses = publishers.get(0).grpcAddresses(); - final Set consumerAddresses = consumers.get(0).grpcAddresses(); - - final Optional publisherAddress = grpcAddresses.stream().findFirst(); - Assertions.assertTrue(publisherAddress.isPresent()); - final Optional consumerAddress = - consumerAddresses.stream().findFirst(); - Assertions.assertTrue(consumerAddress.isPresent()); - - final ManagedChannel publisherChannel = createReadyChannel(publisherAddress.get()); + final InetSocketAddress publisherAddress = findGrpcAddress(cluster, GrpcRole.PUBLISHER); + final InetSocketAddress consumerAddress = findGrpcAddress(cluster, GrpcRole.CONSUMER); - final ManagedChannel consumerChannel = createReadyChannel(consumerAddress.get()); + final ManagedChannel publisherChannel = createReadyChannel(publisherAddress); + final ManagedChannel consumerChannel = createReadyChannel(consumerAddress); final PublisherServiceBlockingStub publisherService = PublisherServiceGrpc.newBlockingStub(publisherChannel); @@ -322,14 +120,7 @@ void testPublishAndConsumeData() { ConsumerServiceGrpc.newStub(consumerChannel); try { - final List users = - Instancio.ofList(User.class) - .size(100) - .generate( - Select.field(User::getName), - g -> g.string().alphaNumeric().allowEmpty().nullable()) - .generate(Select.field(User::getAge), Generators::ints) - .create(); + final List users = generateTestUsers(); Unreliables.retryUntilSuccess( 60, @@ -339,12 +130,9 @@ void testPublishAndConsumeData() { PublishBatchRequest.newBuilder(); for (User user : users) { requestBuilder.addMessages( - BatchRequestMessage.newBuilder() - .setPayload(ByteString.copyFrom(MAPPER.writeValueAsBytes(user)))); + BatchRequestMessage.newBuilder().setPayload(serializeUser(user))); } - final PublishBatchRequest publishRequest = - requestBuilder.setQueue(queueName).build(); - publisherService.publishBatch(publishRequest); + publisherService.publishBatch(requestBuilder.setQueue(queueName).build()); return true; }); @@ -359,15 +147,7 @@ void testPublishAndConsumeData() { @Override public void onNext(SubscriptionNotifications value) { value.getNotificationsList().stream() - .map( - n -> { - try { - return MAPPER.readValue( - n.getMessage().getPayload().toByteArray(), User.class); - } catch (IOException e) { - throw new RuntimeException(e); - } - }) + .map(n -> deserializeUser(n.getMessage().getPayload())) .forEach(result::add); } @@ -382,9 +162,7 @@ public void onCompleted() {} return true; }); - Unreliables.retryUntilTrue( - 60, TimeUnit.SECONDS, () -> new LinkedHashSet<>(users).size() == result.size()); - Assertions.assertEquals(new LinkedHashSet<>(users), result); + assertAllUsersConsumed(users, result); } finally { consumerChannel.shutdownNow(); publisherChannel.shutdownNow(); diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java index 6d66b3aa..5775aec9 100644 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java @@ -5,43 +5,27 @@ package org.testcontainers.containers.integration.tqe; -import java.io.IOException; import java.net.InetSocketAddress; -import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; -import java.util.Arrays; -import java.util.LinkedHashSet; import java.util.List; -import java.util.Optional; import java.util.Set; -import java.util.UUID; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; -import com.google.protobuf.ByteString; import io.grpc.ManagedChannel; import io.grpc.stub.StreamObserver; -import org.instancio.Instancio; -import org.instancio.Select; -import org.instancio.generators.Generators; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.RepeatedTest; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; import org.rnorth.ducttape.unreliables.Unreliables; -import org.testcontainers.containers.ContainerLaunchException; -import org.testcontainers.containers.tqe.GrpcContainer; import org.testcontainers.containers.tqe.GrpcContainer.GrpcRole; import org.testcontainers.containers.tqe.TQE3ClusterImpl; import org.testcontainers.containers.tqe.TQECluster; import org.testcontainers.containers.tqe.configuration.FileTQEConfigurator; import org.testcontainers.containers.tqe.configuration.TQEConfigurator; import org.testcontainers.containers.utils.pojo.User; -import org.testcontainers.shaded.com.fasterxml.jackson.databind.ObjectMapper; import org.testcontainers.utility.DockerImageName; import tarantool.queue_ee.Consumer.SubscriptionRequest; import tarantool.queue_ee.Consumer.SubscriptionStreamRequest; @@ -53,9 +37,7 @@ import tarantool.queue_ee.ProducerOuterClass.ProduceMessage; import tarantool.queue_ee.ProducerOuterClass.ProduceRequest; -class TQE3ClusterImplTest extends AbstractTQETest { - - private static final ObjectMapper MAPPER = new ObjectMapper(); +class TQE3ClusterImplTest extends AbstractTQEClusterTest { private static final DockerImageName IMAGE_NAME = DockerImageName.parse( @@ -80,210 +62,41 @@ protected Path simpleQueueConfig() { return SIMPLE_QUEUE_CONFIG; } - @RepeatedTest(10) - void testMultiplyRestart() throws Exception { - try (TQEConfigurator configurator = - FileTQEConfigurator.tqe3Builder( - IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(SIMPLE_GRPC_CONFIG)) - .build(); - TQECluster cluster = new TQE3ClusterImpl(configurator)) { - cluster.start(); - } + @Override + protected TQECluster createCluster(TQEConfigurator configurator) { + return new TQE3ClusterImpl(configurator); } - @Test - void testRestartMethod() throws Exception { - try (TQEConfigurator configurator = - FileTQEConfigurator.tqe3Builder( - IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(SIMPLE_GRPC_CONFIG)) - .build(); - TQECluster cluster = new TQE3ClusterImpl(configurator)) { - cluster.start(); - cluster.restart(1, TimeUnit.SECONDS, 1, TimeUnit.SECONDS); - } + @Override + protected TQEConfigurator createConfigurator(Path queueConfig, Set grpcConfigs) { + return FileTQEConfigurator.tqe3Builder(IMAGE_NAME, queueConfig, grpcConfigs).build(); } - public static Stream dataForTestInvalidQueueConfigShouldThrow() { - final List invalidConfigs = - Arrays.asList( - // no required test-super user - """ - # Credentials - credentials: - users: - admin: - password: 'secret-cluster-cookie' - roles: [ super ] - replicator: - password: 'secret' - roles: [ replication ] - storage: - roles: [ sharding ] - password: storage - - # advertise configs for all nodes - iproto: - advertise: - peer: - login: replicator - sharding: - login: storage - password: storage - - roles: [ roles.metrics-export ] - # queues configs - roles_cfg: - roles.tqe-storage: - queues: - - name: test - deduplication_mode: keep_latest - disabled_filters_by: [ sharding_key ] - roles.metrics-export: - http: - - listen: 8081 - endpoints: - - format: prometheus - path: '/metrics' - - groups: - routers: - replicasets: - router-1: - sharding: - roles: [ router ] - roles: - - roles.tqe-router - instances: - router: - iproto: - listen: - - uri: router:3301 - storages: - replicasets: - storage-1: - replication: - failover: manual - sharding: - roles: [ storage ] - roles: - - roles.tqe-storage - leader: master - instances: - master: - iproto: - listen: - - uri: master:3301 - net_msg_max: 768 - """, - // no consumer storage to connect from grpc - """ - # Credentials - credentials: - users: - test-super: - password: 'test' - roles: [ super ] - admin: - password: 'secret-cluster-cookie' - roles: [ super ] - replicator: - password: 'secret' - roles: [ replication ] - storage: - roles: [ sharding ] - password: storage - - # advertise configs for all nodes - iproto: - advertise: - peer: - login: replicator - sharding: - login: storage - password: storage - - roles: [ roles.metrics-export ] - # queues configs - roles_cfg: - roles.tqe-storage: - queues: - - name: test - deduplication_mode: keep_latest - disabled_filters_by: [ sharding_key ] - roles.metrics-export: - http: - - listen: 8081 - endpoints: - - format: prometheus - path: '/metrics' - - groups: - routers: - replicasets: - router-1: - sharding: - roles: [ router ] - roles: - - roles.tqe-router - instances: - router: - iproto: - listen: - - uri: router:3301 - """); - - return invalidConfigs.stream() - .map( - s -> { - final Path testConfigPath = TEST_TEMP_DIR.resolve(UUID.randomUUID().toString()); - try { - Files.writeString(testConfigPath, s); - return Arguments.of(testConfigPath); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + @Override + protected TQEConfigurator createConfiguratorWithTimeout( + Path queueConfig, Set grpcConfigs, Duration startupTimeout) { + return FileTQEConfigurator.tqe3Builder(IMAGE_NAME, queueConfig, grpcConfigs) + .withStartupTimeout(startupTimeout) + .build(); } - @ParameterizedTest - @MethodSource("dataForTestInvalidQueueConfigShouldThrow") - void testInvalidQueueConfigShouldThrow(Path queueConfig) { - Assertions.assertThrows( - ContainerLaunchException.class, - () -> { - try (TQEConfigurator configurator = - FileTQEConfigurator.tqe3Builder( - IMAGE_NAME, queueConfig, Set.of(SIMPLE_GRPC_CONFIG)) - .withStartupTimeout(Duration.ofSeconds(5)) - .build(); - TQECluster cluster = new TQE3ClusterImpl(configurator)) { - cluster.start(); - } - }); + @Override + public Stream dataForTestInvalidQueueConfigShouldThrow() { + return loadConfigs( + "tqe3/invalid-config/no-test-super-user.yml", + "tqe3/invalid-config/no-consumer-storage.yml") + .stream() + .map(Arguments::of); } - public static Stream dataForTestInvalidGrpcConfig() { + @Override + public Stream dataForTestInvalidGrpcConfig() { return createInvalidGrpcConfigStream("producer", TEST_TEMP_DIR); } - @ParameterizedTest - @MethodSource("dataForTestInvalidGrpcConfig") - void testInvalidGrpcConfig(Path grpcConfig) { - Assertions.assertThrows( - ContainerLaunchException.class, - () -> { - try (TQEConfigurator configurator = - FileTQEConfigurator.tqe3Builder( - IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(grpcConfig)) - .withStartupTimeout(Duration.ofSeconds(5)) - .build(); - TQECluster cluster = new TQE3ClusterImpl(configurator)) { - cluster.start(); - } - }); - } + // --- TQE3-specific test --- - @RepeatedTest(10) + @RepeatedTest(3) void testPublishAndConsumeData() { Assertions.assertDoesNotThrow( () -> { @@ -296,43 +109,17 @@ void testPublishAndConsumeData() { final String queueName = "test"; - final List> producers = - cluster.grpc().values().stream() - .filter(g -> g.roles().contains(GrpcRole.PRODUCER)) - .toList(); - final List> consumers = - cluster.grpc().values().stream() - .filter(g -> g.roles().contains(GrpcRole.CONSUMER)) - .toList(); - - Assertions.assertFalse(producers.isEmpty()); - Assertions.assertFalse(consumers.isEmpty()); - - final Set grpcAddresses = producers.get(0).grpcAddresses(); - final Set consumerAddresses = consumers.get(0).grpcAddresses(); - - final Optional producerAddress = grpcAddresses.stream().findFirst(); - Assertions.assertTrue(producerAddress.isPresent()); - final Optional consumerAddress = - consumerAddresses.stream().findFirst(); - Assertions.assertTrue(consumerAddress.isPresent()); - - final ManagedChannel producerChannel = createReadyChannel(producerAddress.get()); + final InetSocketAddress producerAddress = findGrpcAddress(cluster, GrpcRole.PRODUCER); + final InetSocketAddress consumerAddress = findGrpcAddress(cluster, GrpcRole.CONSUMER); - final ManagedChannel consumerChannel = createReadyChannel(consumerAddress.get()); + final ManagedChannel producerChannel = createReadyChannel(producerAddress); + final ManagedChannel consumerChannel = createReadyChannel(consumerAddress); final ProducerBlockingStub producer = ProducerGrpc.newBlockingStub(producerChannel); final ConsumerServiceStub consumer = ConsumerServiceGrpc.newStub(consumerChannel); try { - final List users = - Instancio.ofList(User.class) - .size(100) - .generate( - Select.field(User::getName), - g -> g.string().alphaNumeric().allowEmpty().nullable()) - .generate(Select.field(User::getAge), Generators::ints) - .create(); + final List users = generateTestUsers(); Unreliables.retryUntilSuccess( 60, @@ -342,11 +129,9 @@ void testPublishAndConsumeData() { ProduceRequest.newBuilder().setQueue(queueName); for (User user : users) { requestBuilder.addMessages( - ProduceMessage.newBuilder() - .setPayload(ByteString.copyFrom(MAPPER.writeValueAsBytes(user)))); + ProduceMessage.newBuilder().setPayload(serializeUser(user))); } - final ProduceRequest produceRequest = requestBuilder.build(); - producer.produce(produceRequest); + producer.produce(requestBuilder.build()); return true; }); @@ -361,16 +146,7 @@ void testPublishAndConsumeData() { @Override public void onNext(SubscriptionStreamResponse response) { response.getNotifications().getNotificationsList().stream() - .map( - n -> { - try { - return MAPPER.readValue( - n.getMessage().getPayload().toByteArray(), - User.class); - } catch (IOException e) { - throw new RuntimeException(e); - } - }) + .map(n -> deserializeUser(n.getMessage().getPayload())) .forEach(result::add); } @@ -390,9 +166,7 @@ public void onCompleted() {} return true; }); - Unreliables.retryUntilTrue( - 60, TimeUnit.SECONDS, () -> new LinkedHashSet<>(users).size() == result.size()); - Assertions.assertEquals(new LinkedHashSet<>(users), result); + assertAllUsersConsumed(users, result); } finally { consumerChannel.shutdownNow(); producerChannel.shutdownNow(); diff --git a/testcontainers/src/test/resources/tqe2/invalid-config/no-consumer-storage.yml b/testcontainers/src/test/resources/tqe2/invalid-config/no-consumer-storage.yml new file mode 100644 index 00000000..0732bbb2 --- /dev/null +++ b/testcontainers/src/test/resources/tqe2/invalid-config/no-consumer-storage.yml @@ -0,0 +1,52 @@ +# Credentials +credentials: + users: + test-super: + password: 'test' + roles: [ super ] + admin: + password: 'secret-cluster-cookie' + roles: [ super ] + replicator: + password: 'secret' + roles: [ replication ] + storage: + roles: [ sharding ] + password: storage + +# advertise configs for all nodes +iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + password: storage + +roles: [ roles.metrics-export ] +# queues configs +roles_cfg: + app.roles.queue: + queues: + - name: test + deduplication_mode: keep_latest + disabled_filters_by: [ sharding_key ] + roles.metrics-export: + http: + - listen: 8081 + endpoints: + - format: prometheus + path: '/metrics' + +groups: + routers: + replicasets: + r-1: + sharding: + roles: [ router ] + roles: [ app.roles.api ] + instances: + router: + iproto: + listen: + - uri: router:3301 diff --git a/testcontainers/src/test/resources/tqe2/invalid-config/no-test-super-user.yml b/testcontainers/src/test/resources/tqe2/invalid-config/no-test-super-user.yml new file mode 100644 index 00000000..2bc57ef7 --- /dev/null +++ b/testcontainers/src/test/resources/tqe2/invalid-config/no-test-super-user.yml @@ -0,0 +1,63 @@ +# Credentials +credentials: + users: + admin: + password: 'secret-cluster-cookie' + roles: [ super ] + replicator: + password: 'secret' + roles: [ replication ] + storage: + roles: [ sharding ] + password: storage + +# advertise configs for all nodes +iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + password: storage + +roles: [ roles.metrics-export ] +# queues configs +roles_cfg: + app.roles.queue: + queues: + - name: test + deduplication_mode: keep_latest + disabled_filters_by: [ sharding_key ] + roles.metrics-export: + http: + - listen: 8081 + endpoints: + - format: prometheus + path: '/metrics' + +groups: + routers: + replicasets: + r-1: + sharding: + roles: [ router ] + roles: [ app.roles.api ] + instances: + router: + iproto: + listen: + - uri: router:3301 + storages: + replicasets: + shard-1: + replication: + failover: manual + sharding: + roles: [ storage ] + leader: master + instances: + master: + iproto: + listen: + - uri: master:3301 + net_msg_max: 768 diff --git a/testcontainers/src/test/resources/tqe2/invalid-config/router-no-required-roles.yml b/testcontainers/src/test/resources/tqe2/invalid-config/router-no-required-roles.yml new file mode 100644 index 00000000..1614cc27 --- /dev/null +++ b/testcontainers/src/test/resources/tqe2/invalid-config/router-no-required-roles.yml @@ -0,0 +1,61 @@ +# Credentials +credentials: + users: + test-super: + password: 'test' + roles: [ super ] + admin: + password: 'secret-cluster-cookie' + roles: [ super ] + replicator: + password: 'secret' + roles: [ replication ] + storage: + roles: [ sharding ] + password: storage +# advertise configs for all nodes +iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + password: storage +roles: [ roles.metrics-export ] +# queues configs +roles_cfg: + app.roles.queue: + queues: + - name: test + deduplication_mode: keep_latest + disabled_filters_by: [ sharding_key ] + roles.metrics-export: + http: + - listen: 8081 + endpoints: + - format: prometheus + path: '/metrics' +groups: + routers: + replicasets: + r-1: + sharding: + roles: [ router ] + instances: + router: + iproto: + listen: + - uri: router:3301 + storages: + replicasets: + shard-1: + replication: + failover: manual + sharding: + roles: [ storage ] + leader: master + instances: + master: + iproto: + listen: + - uri: master:3301 diff --git a/testcontainers/src/test/resources/tqe2/invalid-config/storage-no-listen-uri.yml b/testcontainers/src/test/resources/tqe2/invalid-config/storage-no-listen-uri.yml new file mode 100644 index 00000000..dd9b9475 --- /dev/null +++ b/testcontainers/src/test/resources/tqe2/invalid-config/storage-no-listen-uri.yml @@ -0,0 +1,61 @@ +# Credentials +credentials: + users: + test-super: + password: 'test' + roles: [ super ] + admin: + password: 'secret-cluster-cookie' + roles: [ super ] + replicator: + password: 'secret' + roles: [ replication ] + storage: + roles: [ sharding ] + password: storage +# advertise configs for all nodes +iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + password: storage +roles: [ roles.metrics-export ] +# queues configs +roles_cfg: + app.roles.queue: + queues: + - name: test + deduplication_mode: keep_latest + disabled_filters_by: [ sharding_key ] + roles.metrics-export: + http: + - listen: 8081 + endpoints: + - format: prometheus + path: '/metrics' +groups: + routers: + replicasets: + r-1: + sharding: + roles: [ router ] + roles: [ app.roles.api ] + instances: + router: + iproto: + listen: + - uri: router:3301 + storages: + replicasets: + shard-1: + replication: + failover: manual + sharding: + roles: [ storage ] + leader: master + instances: + master: + iproto: + net_msg_max: 768 diff --git a/testcontainers/src/test/resources/tqe3/invalid-config/no-consumer-storage.yml b/testcontainers/src/test/resources/tqe3/invalid-config/no-consumer-storage.yml new file mode 100644 index 00000000..1185efe0 --- /dev/null +++ b/testcontainers/src/test/resources/tqe3/invalid-config/no-consumer-storage.yml @@ -0,0 +1,53 @@ +# Credentials +credentials: + users: + test-super: + password: 'test' + roles: [ super ] + admin: + password: 'secret-cluster-cookie' + roles: [ super ] + replicator: + password: 'secret' + roles: [ replication ] + storage: + roles: [ sharding ] + password: storage + +# advertise configs for all nodes +iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + password: storage + +roles: [ roles.metrics-export ] +# queues configs +roles_cfg: + roles.tqe-storage: + queues: + - name: test + deduplication_mode: keep_latest + disabled_filters_by: [ sharding_key ] + roles.metrics-export: + http: + - listen: 8081 + endpoints: + - format: prometheus + path: '/metrics' + +groups: + routers: + replicasets: + router-1: + sharding: + roles: [ router ] + roles: + - roles.tqe-router + instances: + router: + iproto: + listen: + - uri: router:3301 diff --git a/testcontainers/src/test/resources/tqe3/invalid-config/no-test-super-user.yml b/testcontainers/src/test/resources/tqe3/invalid-config/no-test-super-user.yml new file mode 100644 index 00000000..87b9e9dd --- /dev/null +++ b/testcontainers/src/test/resources/tqe3/invalid-config/no-test-super-user.yml @@ -0,0 +1,66 @@ +# Credentials +credentials: + users: + admin: + password: 'secret-cluster-cookie' + roles: [ super ] + replicator: + password: 'secret' + roles: [ replication ] + storage: + roles: [ sharding ] + password: storage + +# advertise configs for all nodes +iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + password: storage + +roles: [ roles.metrics-export ] +# queues configs +roles_cfg: + roles.tqe-storage: + queues: + - name: test + deduplication_mode: keep_latest + disabled_filters_by: [ sharding_key ] + roles.metrics-export: + http: + - listen: 8081 + endpoints: + - format: prometheus + path: '/metrics' + +groups: + routers: + replicasets: + router-1: + sharding: + roles: [ router ] + roles: + - roles.tqe-router + instances: + router: + iproto: + listen: + - uri: router:3301 + storages: + replicasets: + storage-1: + replication: + failover: manual + sharding: + roles: [ storage ] + roles: + - roles.tqe-storage + leader: master + instances: + master: + iproto: + listen: + - uri: master:3301 + net_msg_max: 768 diff --git a/testcontainers/src/test/resources/tqe3/invalid-config/router-no-required-roles.yml b/testcontainers/src/test/resources/tqe3/invalid-config/router-no-required-roles.yml new file mode 100644 index 00000000..ae9818ac --- /dev/null +++ b/testcontainers/src/test/resources/tqe3/invalid-config/router-no-required-roles.yml @@ -0,0 +1,63 @@ +# Credentials +credentials: + users: + test-super: + password: 'test' + roles: [ super ] + admin: + password: 'secret-cluster-cookie' + roles: [ super ] + replicator: + password: 'secret' + roles: [ replication ] + storage: + roles: [ sharding ] + password: storage +# advertise configs for all nodes +iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + password: storage +roles: [ roles.metrics-export ] +# queues configs +roles_cfg: + roles.tqe-storage: + queues: + - name: test + deduplication_mode: keep_latest + disabled_filters_by: [ sharding_key ] + roles.metrics-export: + http: + - listen: 8081 + endpoints: + - format: prometheus + path: '/metrics' +groups: + routers: + replicasets: + router-1: + sharding: + roles: [ router ] + instances: + router: + iproto: + listen: + - uri: router:3301 + storages: + replicasets: + storage-1: + replication: + failover: manual + sharding: + roles: [ storage ] + roles: + - roles.tqe-storage + leader: master + instances: + master: + iproto: + listen: + - uri: master:3301 diff --git a/testcontainers/src/test/resources/tqe3/invalid-config/storage-no-listen-uri.yml b/testcontainers/src/test/resources/tqe3/invalid-config/storage-no-listen-uri.yml new file mode 100644 index 00000000..49033720 --- /dev/null +++ b/testcontainers/src/test/resources/tqe3/invalid-config/storage-no-listen-uri.yml @@ -0,0 +1,62 @@ +# Credentials +credentials: + users: + test-super: + password: 'test' + roles: [ super ] + admin: + password: 'secret-cluster-cookie' + roles: [ super ] + replicator: + password: 'secret' + roles: [ replication ] + storage: + roles: [ sharding ] + password: storage +# advertise configs for all nodes +iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + password: storage +roles: [ roles.metrics-export ] +# queues configs +roles_cfg: + roles.tqe-storage: + queues: + - name: test + deduplication_mode: keep_latest + disabled_filters_by: [ sharding_key ] + roles.metrics-export: + http: + - listen: 8081 + endpoints: + - format: prometheus + path: '/metrics' +groups: + routers: + replicasets: + router-1: + sharding: + roles: [ router ] + roles: + - roles.tqe-router + instances: + router: + iproto: + listen: + - uri: router:3301 + storages: + replicasets: + storage-1: + replication: + failover: manual + sharding: + roles: [ storage ] + leader: master + instances: + master: + iproto: + net_msg_max: 768 From 26004ad7bb7cb67d3f9540768a41082bd7c8e1d1 Mon Sep 17 00:00:00 2001 From: Dmitry Kasimovskiy Date: Thu, 4 Jun 2026 19:05:05 +0300 Subject: [PATCH 4/5] refactor(testcontainers): replace abstract test classes with parameterized tests Replace AbstractTQETest/AbstractTQEClusterTest hierarchy with a flat structure using @ParameterizedTest. Common tests for TQE2/TQE3 now run via TQETestHelper.TQETestParams parameterization, eliminating inheritance and duplication. - Add TQETestHelper with TQETestParams record and shared helpers - Add TQEClusterTest with parameterized common cluster tests - Add FileTQEConfiguratorTest with parameterized configurator tests - Simplify TQE2/TQE3ClusterImplTest to only version-specific tests - Remove AbstractTQETest, AbstractTQEClusterTest, FileTQE2/3ConfiguratorTest, and invalid-config resource files - Inline YAML configs restored as in original tests Co-Authored-By: Claude Opus 4.7 --- .../tqe/AbstractTQEClusterTest.java | 87 ----- .../integration/tqe/AbstractTQETest.java | 270 -------------- .../tqe/FileTQE2ConfiguratorTest.java | 100 ----- .../tqe/FileTQE3ConfiguratorTest.java | 100 ----- .../tqe/FileTQEConfiguratorTest.java | 226 +++++++++++ .../integration/tqe/TQE2ClusterImplTest.java | 138 ++++--- .../integration/tqe/TQE3ClusterImplTest.java | 132 +++---- .../integration/tqe/TQEClusterTest.java | 351 ++++++++++++++++++ .../integration/tqe/TQETestHelper.java | 138 +++++++ .../invalid-config/no-consumer-storage.yml | 52 --- .../invalid-config/no-test-super-user.yml | 63 ---- .../router-no-required-roles.yml | 61 --- .../invalid-config/storage-no-listen-uri.yml | 61 --- .../invalid-config/no-consumer-storage.yml | 53 --- .../invalid-config/no-test-super-user.yml | 66 ---- .../router-no-required-roles.yml | 63 ---- .../invalid-config/storage-no-listen-uri.yml | 62 ---- 17 files changed, 849 insertions(+), 1174 deletions(-) delete mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQEClusterTest.java delete mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQETest.java delete mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE2ConfiguratorTest.java delete mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE3ConfiguratorTest.java create mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQEConfiguratorTest.java create mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterTest.java create mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQETestHelper.java delete mode 100644 testcontainers/src/test/resources/tqe2/invalid-config/no-consumer-storage.yml delete mode 100644 testcontainers/src/test/resources/tqe2/invalid-config/no-test-super-user.yml delete mode 100644 testcontainers/src/test/resources/tqe2/invalid-config/router-no-required-roles.yml delete mode 100644 testcontainers/src/test/resources/tqe2/invalid-config/storage-no-listen-uri.yml delete mode 100644 testcontainers/src/test/resources/tqe3/invalid-config/no-consumer-storage.yml delete mode 100644 testcontainers/src/test/resources/tqe3/invalid-config/no-test-super-user.yml delete mode 100644 testcontainers/src/test/resources/tqe3/invalid-config/router-no-required-roles.yml delete mode 100644 testcontainers/src/test/resources/tqe3/invalid-config/storage-no-listen-uri.yml diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQEClusterTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQEClusterTest.java deleted file mode 100644 index 8d09f292..00000000 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQEClusterTest.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY - * All Rights Reserved. - */ - -package org.testcontainers.containers.integration.tqe; - -import java.nio.file.Path; -import java.time.Duration; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.RepeatedTest; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.testcontainers.containers.ContainerLaunchException; -import org.testcontainers.containers.tqe.TQECluster; -import org.testcontainers.containers.tqe.configuration.TQEConfigurator; - -abstract class AbstractTQEClusterTest extends AbstractTQETest { - - protected abstract TQECluster createCluster(TQEConfigurator configurator); - - protected abstract TQEConfigurator createConfigurator(Path queueConfig, Set grpcConfigs); - - protected abstract TQEConfigurator createConfiguratorWithTimeout( - Path queueConfig, Set grpcConfigs, Duration startupTimeout); - - // --- Common cluster tests --- - - @RepeatedTest(3) - void testMultiplyRestart() throws Exception { - try (TQEConfigurator configurator = - createConfigurator(simpleQueueConfig(), Set.of(simpleGrpcConfig())); - TQECluster cluster = createCluster(configurator)) { - cluster.start(); - } - } - - @Test - void testRestartMethod() throws Exception { - try (TQEConfigurator configurator = - createConfigurator(simpleQueueConfig(), Set.of(simpleGrpcConfig())); - TQECluster cluster = createCluster(configurator)) { - cluster.start(); - cluster.restart(1, TimeUnit.SECONDS, 1, TimeUnit.SECONDS); - } - } - - public abstract Stream dataForTestInvalidQueueConfigShouldThrow(); - - @ParameterizedTest - @MethodSource("dataForTestInvalidQueueConfigShouldThrow") - void testInvalidQueueConfigShouldThrow(Path queueConfig) { - Assertions.assertThrows( - ContainerLaunchException.class, - () -> { - try (TQEConfigurator configurator = - createConfiguratorWithTimeout( - queueConfig, Set.of(simpleGrpcConfig()), Duration.ofSeconds(5)); - TQECluster cluster = createCluster(configurator)) { - cluster.start(); - } - }); - } - - public abstract Stream dataForTestInvalidGrpcConfig(); - - @ParameterizedTest - @MethodSource("dataForTestInvalidGrpcConfig") - void testInvalidGrpcConfig(Path grpcConfig) { - Assertions.assertThrows( - ContainerLaunchException.class, - () -> { - try (TQEConfigurator configurator = - createConfiguratorWithTimeout( - simpleQueueConfig(), Set.of(grpcConfig), Duration.ofSeconds(5)); - TQECluster cluster = createCluster(configurator)) { - cluster.start(); - } - }); - } -} diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQETest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQETest.java deleted file mode 100644 index 2caf3881..00000000 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/AbstractTQETest.java +++ /dev/null @@ -1,270 +0,0 @@ -/* - * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY - * All Rights Reserved. - */ - -package org.testcontainers.containers.integration.tqe; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.protobuf.ByteString; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import org.instancio.Instancio; -import org.instancio.Select; -import org.instancio.generators.Generators; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.provider.Arguments; -import org.rnorth.ducttape.unreliables.Unreliables; -import org.testcontainers.containers.tqe.GrpcContainer; -import org.testcontainers.containers.tqe.GrpcContainer.GrpcRole; -import org.testcontainers.containers.tqe.TQECluster; -import org.testcontainers.containers.utils.pojo.User; -import org.testcontainers.utility.DockerImageName; - -abstract class AbstractTQETest { - - @TempDir protected static Path TEST_TEMP_DIR; - - protected static final ObjectMapper MAPPER = new ObjectMapper(); - - protected abstract DockerImageName imageName(); - - protected abstract Path simpleGrpcConfig(); - - protected abstract Path simpleQueueConfig(); - - protected static Path loadConfig(String resourcePath) { - try { - return Paths.get( - Objects.requireNonNull(AbstractTQETest.class.getClassLoader().getResource(resourcePath)) - .toURI()); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } - - protected static List loadConfigs(String... resourcePaths) { - return Arrays.stream(resourcePaths).map(AbstractTQETest::loadConfig).toList(); - } - - protected static ManagedChannel createReadyChannel(InetSocketAddress address) { - return Unreliables.retryUntilSuccess( - 60, - TimeUnit.SECONDS, - () -> { - ManagedChannel ch = - ManagedChannelBuilder.forAddress(address.getHostName(), address.getPort()) - .usePlaintext() - .maxInboundMessageSize(16 * 1024 * 1024) - .keepAliveTime(30, TimeUnit.SECONDS) - .keepAliveTimeout(5, TimeUnit.SECONDS) - .keepAliveWithoutCalls(true) - .build(); - - ch.getState(true); - Unreliables.retryUntilTrue( - 5, - TimeUnit.SECONDS, - () -> { - io.grpc.ConnectivityState state = ch.getState(false); - if (state == io.grpc.ConnectivityState.READY) { - return true; - } - ch.resetConnectBackoff(); - Thread.sleep(100); - return false; - }); - return ch; - }); - } - - protected static List invalidGrpcConfigs(String producerRoleName) { - return Arrays.asList( - // unknown host - """ - core_port: 1111 - grpc_listen: - - uri: 'tcp://0.0.0.0:18182' - - %s: - enabled: true - tarantool: - user: test-super - pass: test - connections: - routers: - - "unknown:3301" - - consumer: - enabled: true - tarantool: - user: test-super - pass: test - connections: - storage: - - "master:3301" - """ - .formatted(producerRoleName), - // no consumers and producers - """ - core_port: 1111 - grpc_listen: - - uri: 'tcp://0.0.0.0:18182' - - %s: - enabled: false - tarantool: - user: test-super - pass: test - connections: - routers: - - "router:3301" - - consumer: - enabled: false - tarantool: - user: test-super - pass: test - connections: - storage: - - "master:3301" - """ - .formatted(producerRoleName), - // no core_port parameter - """ - grpc_listen: - - uri: 'tcp://0.0.0.0:18182' - - %s: - enabled: true - tarantool: - user: test-super - pass: test - connections: - routers: - - "router:3301" - - consumer: - enabled: true - tarantool: - user: test-super - pass: test - connections: - storage: - - "master:3301" - """ - .formatted(producerRoleName), - // no listen.uri parameter - """ - core_port: 1111 - - %s: - enabled: true - tarantool: - user: test-super - pass: test - connections: - routers: - - "router:3301" - - consumer: - enabled: true - tarantool: - user: test-super - pass: test - connections: - storage: - - "master:3301" - """ - .formatted(producerRoleName)); - } - - protected static Stream createInvalidGrpcConfigStream( - String producerRoleName, Path tempDir) { - return invalidGrpcConfigs(producerRoleName).stream() - .map( - s -> { - final Path testConfigPath = tempDir.resolve(UUID.randomUUID() + ".yml"); - try { - Files.writeString(testConfigPath, s); - return testConfigPath; - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } - - protected static Stream createInvalidConfigsPathsStream( - Path simpleQueueConfig, Path simpleGrpcConfig, Path tempDir) { - return Stream.of( - // invalid grpc configs - // null - Arguments.of(simpleQueueConfig, null), - // empty - Arguments.of(simpleQueueConfig, Set.of()), - // non regular - Arguments.of(simpleQueueConfig, Set.of(tempDir)), - - // invalid queue config - Arguments.of(null, Set.of(simpleGrpcConfig)), - Arguments.of(tempDir, Set.of(simpleGrpcConfig))); - } - - protected static List generateTestUsers() { - return Instancio.ofList(User.class) - .size(100) - .generate( - Select.field(User::getName), g -> g.string().alphaNumeric().allowEmpty().nullable()) - .generate(Select.field(User::getAge), Generators::ints) - .create(); - } - - protected static InetSocketAddress findGrpcAddress(TQECluster cluster, GrpcRole role) { - final List> containers = - cluster.grpc().values().stream().filter(g -> g.roles().contains(role)).toList(); - Assertions.assertFalse(containers.isEmpty(), "No gRPC container with role: " + role); - final Optional address = - containers.get(0).grpcAddresses().stream().findFirst(); - Assertions.assertTrue(address.isPresent(), "No gRPC address for role: " + role); - return address.get(); - } - - protected static User deserializeUser(ByteString payload) { - try { - return MAPPER.readValue(payload.toByteArray(), User.class); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - protected static ByteString serializeUser(User user) { - try { - return ByteString.copyFrom(MAPPER.writeValueAsBytes(user)); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - protected static void assertAllUsersConsumed(List users, Set result) { - Unreliables.retryUntilTrue( - 60, TimeUnit.SECONDS, () -> new LinkedHashSet<>(users).size() == result.size()); - Assertions.assertEquals(new LinkedHashSet<>(users), result); - } -} diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE2ConfiguratorTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE2ConfiguratorTest.java deleted file mode 100644 index 8d39f2c1..00000000 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE2ConfiguratorTest.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY - * All Rights Reserved. - */ - -package org.testcontainers.containers.integration.tqe; - -import java.nio.file.Path; -import java.util.Set; -import java.util.stream.Stream; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.testcontainers.containers.ContainerLaunchException; -import org.testcontainers.containers.tqe.configuration.FileTQEConfigurator; -import org.testcontainers.containers.tqe.configuration.TQEConfigurator; -import org.testcontainers.lifecycle.Startable; -import org.testcontainers.utility.DockerImageName; - -class FileTQE2ConfiguratorTest extends AbstractTQETest { - - private static final DockerImageName IMAGE_NAME = - DockerImageName.parse( - System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") - + "tarantool/message-queue-ee:2.5.3"); - - private static final Path SIMPLE_GRPC_CONFIG = loadConfig("tqe2/simple-config/simple-grpc.yml"); - private static final Path SIMPLE_QUEUE_CONFIG = loadConfig("tqe2/simple-config/simple-queue.yml"); - - @Override - protected DockerImageName imageName() { - return IMAGE_NAME; - } - - @Override - protected Path simpleGrpcConfig() { - return SIMPLE_GRPC_CONFIG; - } - - @Override - protected Path simpleQueueConfig() { - return SIMPLE_QUEUE_CONFIG; - } - - @Test - void simpleConfiguration() { - try (TQEConfigurator configurator = - FileTQEConfigurator.tqe2Builder( - imageName(), simpleQueueConfig(), Set.of(simpleGrpcConfig())) - .build()) { - configurator.queue().values().parallelStream().forEach(Startable::start); - // TQE 2.x does not require explicit configure() — the router role bootstraps automatically - configurator.grpc().values().parallelStream().forEach(Startable::start); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public static Stream dataForTestInvalidQueueConfigShouldThrow() { - return loadConfigs( - "tqe2/invalid-config/router-no-required-roles.yml", - "tqe2/invalid-config/storage-no-listen-uri.yml") - .stream() - .map(Arguments::of); - } - - @ParameterizedTest - @MethodSource("dataForTestInvalidQueueConfigShouldThrow") - void testInvalidQueueConfig(Path invalidQueueConfig) { - Assertions.assertThrows( - ContainerLaunchException.class, - () -> { - try (FileTQEConfigurator c = - FileTQEConfigurator.tqe2Builder( - imageName(), invalidQueueConfig, Set.of(simpleGrpcConfig())) - .build()) { - c.queue(); - } - }); - } - - public static Stream dataForTestInvalidConfigsPaths() { - return createInvalidConfigsPathsStream(SIMPLE_QUEUE_CONFIG, SIMPLE_GRPC_CONFIG, TEST_TEMP_DIR); - } - - @ParameterizedTest - @MethodSource("dataForTestInvalidConfigsPaths") - void testInvalidConfigsPaths(Path invalidGrpcConfig, Set invalidQueueConfigs) { - Assertions.assertThrows( - IllegalArgumentException.class, - () -> { - try (FileTQEConfigurator c = - FileTQEConfigurator.tqe2Builder(imageName(), invalidGrpcConfig, invalidQueueConfigs) - .build()) {} - }); - } -} diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE3ConfiguratorTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE3ConfiguratorTest.java deleted file mode 100644 index 02cc932e..00000000 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQE3ConfiguratorTest.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY - * All Rights Reserved. - */ - -package org.testcontainers.containers.integration.tqe; - -import java.nio.file.Path; -import java.util.Set; -import java.util.stream.Stream; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.testcontainers.containers.ContainerLaunchException; -import org.testcontainers.containers.tqe.configuration.FileTQEConfigurator; -import org.testcontainers.containers.tqe.configuration.TQEConfigurator; -import org.testcontainers.lifecycle.Startable; -import org.testcontainers.utility.DockerImageName; - -class FileTQE3ConfiguratorTest extends AbstractTQETest { - - private static final DockerImageName IMAGE_NAME = - DockerImageName.parse( - System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") - + "tarantool/message-queue-ee:v3.5.0"); - - private static final Path SIMPLE_GRPC_CONFIG = loadConfig("tqe3/simple-config/simple-grpc.yml"); - private static final Path SIMPLE_QUEUE_CONFIG = loadConfig("tqe3/simple-config/simple-queue.yml"); - - @Override - protected DockerImageName imageName() { - return IMAGE_NAME; - } - - @Override - protected Path simpleGrpcConfig() { - return SIMPLE_GRPC_CONFIG; - } - - @Override - protected Path simpleQueueConfig() { - return SIMPLE_QUEUE_CONFIG; - } - - @Test - void simpleConfiguration() { - try (TQEConfigurator configurator = - FileTQEConfigurator.tqe3Builder( - imageName(), simpleQueueConfig(), Set.of(simpleGrpcConfig())) - .build()) { - configurator.queue().values().parallelStream().forEach(Startable::start); - configurator.configure(); - configurator.grpc().values().parallelStream().forEach(Startable::start); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public static Stream dataForTestInvalidQueueConfigShouldThrow() { - return loadConfigs( - "tqe3/invalid-config/router-no-required-roles.yml", - "tqe3/invalid-config/storage-no-listen-uri.yml") - .stream() - .map(Arguments::of); - } - - @ParameterizedTest - @MethodSource("dataForTestInvalidQueueConfigShouldThrow") - void testInvalidQueueConfig(Path invalidQueueConfig) { - Assertions.assertThrows( - ContainerLaunchException.class, - () -> { - try (FileTQEConfigurator c = - FileTQEConfigurator.tqe3Builder( - imageName(), invalidQueueConfig, Set.of(simpleGrpcConfig())) - .build()) { - c.queue(); - } - }); - } - - public static Stream dataForTestInvalidConfigsPaths() { - return createInvalidConfigsPathsStream(SIMPLE_QUEUE_CONFIG, SIMPLE_GRPC_CONFIG, TEST_TEMP_DIR); - } - - @ParameterizedTest - @MethodSource("dataForTestInvalidConfigsPaths") - void testInvalidConfigsPaths(Path invalidGrpcConfig, Set invalidQueueConfigs) { - Assertions.assertThrows( - IllegalArgumentException.class, - () -> { - try (FileTQEConfigurator c = - FileTQEConfigurator.tqe3Builder(imageName(), invalidGrpcConfig, invalidQueueConfigs) - .build()) {} - }); - } -} diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQEConfiguratorTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQEConfiguratorTest.java new file mode 100644 index 00000000..1d001d4b --- /dev/null +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQEConfiguratorTest.java @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY + * All Rights Reserved. + */ + +package org.testcontainers.containers.integration.tqe; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.testcontainers.containers.ContainerLaunchException; +import org.testcontainers.containers.tqe.configuration.FileTQEConfigurator; +import org.testcontainers.containers.tqe.configuration.TQEConfigurator; +import org.testcontainers.lifecycle.Startable; + +class FileTQEConfiguratorTest { + + @ParameterizedTest + @MethodSource("tqeVersions") + void simpleConfiguration(TQETestHelper.TQETestParams p) throws Exception { + try (TQEConfigurator configurator = + p.configuratorBuilderFactory().apply(p.queueConfig(), Set.of(p.grpcConfig())).build()) { + configurator.queue().values().parallelStream().forEach(Startable::start); + if (p.requiresConfigure()) { + configurator.configure(); + } + configurator.grpc().values().parallelStream().forEach(Startable::start); + } + } + + public static Stream dataForTestInvalidQueueConfig() { + return Stream.of( + // router have no required roles + """ + # Credentials + credentials: + users: + test-super: + password: 'test' + roles: [ super ] + admin: + password: 'secret-cluster-cookie' + roles: [ super ] + replicator: + password: 'secret' + roles: [ replication ] + storage: + roles: [ sharding ] + password: storage + # advertise configs for all nodes + iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + password: storage + roles: [ roles.metrics-export ] + # queues configs + roles_cfg: + app.roles.queue: + queues: + - name: test + deduplication_mode: keep_latest + disabled_filters_by: [ sharding_key ] + roles.metrics-export: + http: + - listen: 8081 + endpoints: + - format: prometheus + path: '/metrics' + groups: + routers: + replicasets: + r-1: + sharding: + roles: [ router ] + instances: + router: + iproto: + listen: + - uri: router:3301 + storages: + replicasets: + shard-1: + replication: + failover: manual + sharding: + roles: [ storage ] + leader: master + instances: + master: + iproto: + listen: + - uri: master:3301 + """, + """ + # Credentials + credentials: + users: + test-super: + password: 'test' + roles: [ super ] + admin: + password: 'secret-cluster-cookie' + roles: [ super ] + replicator: + password: 'secret' + roles: [ replication ] + storage: + roles: [ sharding ] + password: storage + # advertise configs for all nodes + iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + password: storage + roles: [ roles.metrics-export ] + # queues configs + roles_cfg: + app.roles.queue: + queues: + - name: test + deduplication_mode: keep_latest + disabled_filters_by: [ sharding_key ] + roles.metrics-export: + http: + - listen: 8081 + endpoints: + - format: prometheus + path: '/metrics' + groups: + routers: + replicasets: + r-1: + sharding: + roles: [ router ] + roles: [ app.roles.api ] + instances: + router: + iproto: + listen: + - uri: router:3301 + storages: + replicasets: + shard-1: + replication: + failover: manual + sharding: + roles: [ storage ] + leader: master + instances: + master: + iproto: + net_msg_max: 768 + """); + } + + @ParameterizedTest + @MethodSource("dataForTestInvalidQueueConfig") + void testInvalidQueueConfig(String invalidQueueConfig) throws IOException { + for (TQETestHelper.TQETestParams p : TQETestHelper.tqeVersions().toList()) { + final Path invalidConfigPath = + TQETestHelper.TEST_TEMP_DIR.resolve(UUID.randomUUID().toString()); + Files.writeString(invalidConfigPath, invalidQueueConfig); + + Assertions.assertThrows( + ContainerLaunchException.class, + () -> { + try (FileTQEConfigurator c = + p.configuratorBuilderFactory() + .apply(invalidConfigPath, Set.of(p.grpcConfig())) + .build()) {} + }); + } + } + + public static Stream dataForTestInvalidConfigsPaths() { + return TQETestHelper.tqeVersions() + .flatMap( + (TQETestHelper.TQETestParams p) -> { + Path qc = p.queueConfig(); + Path gc = p.grpcConfig(); + return Stream.of( + // invalid grpc configs + // null + Arguments.of(p, qc, null), + // empty + Arguments.of(p, qc, Set.of()), + // non regular + Arguments.of(p, qc, Set.of(TQETestHelper.TEST_TEMP_DIR)), + // invalid queue config + Arguments.of(p, (Path) null, Set.of(gc)), + Arguments.of(p, TQETestHelper.TEST_TEMP_DIR, Set.of(gc))); + }); + } + + @ParameterizedTest + @MethodSource("dataForTestInvalidConfigsPaths") + void testInvalidConfigsPaths( + TQETestHelper.TQETestParams p, Path invalidGrpcConfig, Set invalidQueueConfigs) { + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + try (FileTQEConfigurator c = + p.configuratorBuilderFactory() + .apply(invalidGrpcConfig, invalidQueueConfigs) + .build()) {} + }); + } + + static Stream tqeVersions() { + return TQETestHelper.tqeVersions(); + } +} diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java index ec0b8e1b..526a08e2 100644 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java @@ -5,21 +5,27 @@ package org.testcontainers.containers.integration.tqe; +import java.io.IOException; import java.net.InetSocketAddress; import java.nio.file.Path; -import java.time.Duration; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.protobuf.ByteString; import io.grpc.ManagedChannel; import io.grpc.stub.StreamObserver; +import org.instancio.Instancio; +import org.instancio.Select; +import org.instancio.generators.Generators; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.RepeatedTest; -import org.junit.jupiter.params.provider.Arguments; import org.rnorth.ducttape.unreliables.Unreliables; +import org.testcontainers.containers.tqe.GrpcContainer; import org.testcontainers.containers.tqe.GrpcContainer.GrpcRole; import org.testcontainers.containers.tqe.TQE2ClusterImpl; import org.testcontainers.containers.tqe.TQECluster; @@ -36,69 +42,24 @@ import tarantool.queue_ee.v2.PublisherServiceGrpc; import tarantool.queue_ee.v2.PublisherServiceGrpc.PublisherServiceBlockingStub; -class TQE2ClusterImplTest extends AbstractTQEClusterTest { +class TQE2ClusterImplTest { private static final DockerImageName IMAGE_NAME = DockerImageName.parse( System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") + "tarantool/message-queue-ee:2.5.3"); - private static final Path SIMPLE_GRPC_CONFIG = loadConfig("tqe2/simple-config/simple-grpc.yml"); - private static final Path SIMPLE_QUEUE_CONFIG = loadConfig("tqe2/simple-config/simple-queue.yml"); + private static final Path SIMPLE_GRPC_CONFIG = + TQETestHelper.loadConfig("tqe2/simple-config/simple-grpc.yml"); + private static final Path SIMPLE_QUEUE_CONFIG = + TQETestHelper.loadConfig("tqe2/simple-config/simple-queue.yml"); - @Override - protected DockerImageName imageName() { - return IMAGE_NAME; - } - - @Override - protected Path simpleGrpcConfig() { - return SIMPLE_GRPC_CONFIG; - } - - @Override - protected Path simpleQueueConfig() { - return SIMPLE_QUEUE_CONFIG; - } - - @Override - protected TQECluster createCluster(TQEConfigurator configurator) { - return new TQE2ClusterImpl(configurator); - } - - @Override - protected TQEConfigurator createConfigurator(Path queueConfig, Set grpcConfigs) { - return FileTQEConfigurator.tqe2Builder(IMAGE_NAME, queueConfig, grpcConfigs).build(); - } - - @Override - protected TQEConfigurator createConfiguratorWithTimeout( - Path queueConfig, Set grpcConfigs, Duration startupTimeout) { - return FileTQEConfigurator.tqe2Builder(IMAGE_NAME, queueConfig, grpcConfigs) - .withStartupTimeout(startupTimeout) - .build(); - } - - @Override - public Stream dataForTestInvalidQueueConfigShouldThrow() { - return loadConfigs( - "tqe2/invalid-config/no-test-super-user.yml", - "tqe2/invalid-config/no-consumer-storage.yml") - .stream() - .map(Arguments::of); - } - - @Override - public Stream dataForTestInvalidGrpcConfig() { - return createInvalidGrpcConfigStream("publisher", TEST_TEMP_DIR); - } - - // --- TQE2-specific test --- - - @RepeatedTest(3) + @RepeatedTest(10) void testPublishAndConsumeData() { Assertions.assertDoesNotThrow( () -> { + final ObjectMapper MAPPER = new ObjectMapper(); + try (TQEConfigurator configurator = FileTQEConfigurator.tqe2Builder( IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(SIMPLE_GRPC_CONFIG)) @@ -108,19 +69,45 @@ void testPublishAndConsumeData() { final String queueName = "test"; - final InetSocketAddress publisherAddress = findGrpcAddress(cluster, GrpcRole.PUBLISHER); - final InetSocketAddress consumerAddress = findGrpcAddress(cluster, GrpcRole.CONSUMER); + final List> publishers = + cluster.grpc().values().stream() + .filter(g -> g.roles().contains(GrpcRole.PUBLISHER)) + .toList(); + final List> consumers = + cluster.grpc().values().stream() + .filter(g -> g.roles().contains(GrpcRole.CONSUMER)) + .toList(); + + Assertions.assertFalse(publishers.isEmpty()); + Assertions.assertFalse(consumers.isEmpty()); + + final Set grpcAddresses = publishers.get(0).grpcAddresses(); + final Set consumerAddresses = consumers.get(0).grpcAddresses(); - final ManagedChannel publisherChannel = createReadyChannel(publisherAddress); - final ManagedChannel consumerChannel = createReadyChannel(consumerAddress); + final Optional publisherAddress = grpcAddresses.stream().findFirst(); + Assertions.assertTrue(publisherAddress.isPresent()); + final Optional consumerAddress = + consumerAddresses.stream().findFirst(); + Assertions.assertTrue(consumerAddress.isPresent()); - final PublisherServiceBlockingStub publisherService = - PublisherServiceGrpc.newBlockingStub(publisherChannel); - final ConsumerServiceStub consumerService = - ConsumerServiceGrpc.newStub(consumerChannel); + final ManagedChannel publisherChannel = + TQETestHelper.createReadyChannel(publisherAddress.get()); + final ManagedChannel consumerChannel = + TQETestHelper.createReadyChannel(consumerAddress.get()); try { - final List users = generateTestUsers(); + final PublisherServiceBlockingStub pService = + PublisherServiceGrpc.newBlockingStub(publisherChannel); + final ConsumerServiceStub cService = ConsumerServiceGrpc.newStub(consumerChannel); + + final List users = + Instancio.ofList(User.class) + .size(100) + .generate( + Select.field(User::getName), + g -> g.string().alphaNumeric().allowEmpty().nullable()) + .generate(Select.field(User::getAge), Generators::ints) + .create(); Unreliables.retryUntilSuccess( 60, @@ -130,9 +117,10 @@ void testPublishAndConsumeData() { PublishBatchRequest.newBuilder(); for (User user : users) { requestBuilder.addMessages( - BatchRequestMessage.newBuilder().setPayload(serializeUser(user))); + BatchRequestMessage.newBuilder() + .setPayload(ByteString.copyFrom(MAPPER.writeValueAsBytes(user)))); } - publisherService.publishBatch(requestBuilder.setQueue(queueName).build()); + pService.publishBatch(requestBuilder.setQueue(queueName).build()); return true; }); @@ -141,13 +129,21 @@ void testPublishAndConsumeData() { 60, TimeUnit.SECONDS, () -> { - consumerService.subscribe( + cService.subscribe( SubscriptionRequest.newBuilder().setCursor("").setQueue(queueName).build(), new StreamObserver<>() { @Override public void onNext(SubscriptionNotifications value) { value.getNotificationsList().stream() - .map(n -> deserializeUser(n.getMessage().getPayload())) + .map( + n -> { + try { + return MAPPER.readValue( + n.getMessage().getPayload().toByteArray(), User.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) .forEach(result::add); } @@ -162,7 +158,9 @@ public void onCompleted() {} return true; }); - assertAllUsersConsumed(users, result); + Unreliables.retryUntilTrue( + 60, TimeUnit.SECONDS, () -> new LinkedHashSet<>(users).size() == result.size()); + Assertions.assertEquals(new LinkedHashSet<>(users), result); } finally { consumerChannel.shutdownNow(); publisherChannel.shutdownNow(); diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java index 5775aec9..70c4f883 100644 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java @@ -5,21 +5,27 @@ package org.testcontainers.containers.integration.tqe; +import java.io.IOException; import java.net.InetSocketAddress; import java.nio.file.Path; -import java.time.Duration; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.protobuf.ByteString; import io.grpc.ManagedChannel; import io.grpc.stub.StreamObserver; +import org.instancio.Instancio; +import org.instancio.Select; +import org.instancio.generators.Generators; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.RepeatedTest; -import org.junit.jupiter.params.provider.Arguments; import org.rnorth.ducttape.unreliables.Unreliables; +import org.testcontainers.containers.tqe.GrpcContainer; import org.testcontainers.containers.tqe.GrpcContainer.GrpcRole; import org.testcontainers.containers.tqe.TQE3ClusterImpl; import org.testcontainers.containers.tqe.TQECluster; @@ -37,69 +43,24 @@ import tarantool.queue_ee.ProducerOuterClass.ProduceMessage; import tarantool.queue_ee.ProducerOuterClass.ProduceRequest; -class TQE3ClusterImplTest extends AbstractTQEClusterTest { +class TQE3ClusterImplTest { private static final DockerImageName IMAGE_NAME = DockerImageName.parse( System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") + "tarantool/message-queue-ee:v3.5.0"); - private static final Path SIMPLE_GRPC_CONFIG = loadConfig("tqe3/simple-config/simple-grpc.yml"); - private static final Path SIMPLE_QUEUE_CONFIG = loadConfig("tqe3/simple-config/simple-queue.yml"); + private static final Path SIMPLE_GRPC_CONFIG = + TQETestHelper.loadConfig("tqe3/simple-config/simple-grpc.yml"); + private static final Path SIMPLE_QUEUE_CONFIG = + TQETestHelper.loadConfig("tqe3/simple-config/simple-queue.yml"); - @Override - protected DockerImageName imageName() { - return IMAGE_NAME; - } - - @Override - protected Path simpleGrpcConfig() { - return SIMPLE_GRPC_CONFIG; - } - - @Override - protected Path simpleQueueConfig() { - return SIMPLE_QUEUE_CONFIG; - } - - @Override - protected TQECluster createCluster(TQEConfigurator configurator) { - return new TQE3ClusterImpl(configurator); - } - - @Override - protected TQEConfigurator createConfigurator(Path queueConfig, Set grpcConfigs) { - return FileTQEConfigurator.tqe3Builder(IMAGE_NAME, queueConfig, grpcConfigs).build(); - } - - @Override - protected TQEConfigurator createConfiguratorWithTimeout( - Path queueConfig, Set grpcConfigs, Duration startupTimeout) { - return FileTQEConfigurator.tqe3Builder(IMAGE_NAME, queueConfig, grpcConfigs) - .withStartupTimeout(startupTimeout) - .build(); - } - - @Override - public Stream dataForTestInvalidQueueConfigShouldThrow() { - return loadConfigs( - "tqe3/invalid-config/no-test-super-user.yml", - "tqe3/invalid-config/no-consumer-storage.yml") - .stream() - .map(Arguments::of); - } - - @Override - public Stream dataForTestInvalidGrpcConfig() { - return createInvalidGrpcConfigStream("producer", TEST_TEMP_DIR); - } - - // --- TQE3-specific test --- - - @RepeatedTest(3) + @RepeatedTest(10) void testPublishAndConsumeData() { Assertions.assertDoesNotThrow( () -> { + final ObjectMapper MAPPER = new ObjectMapper(); + try (TQEConfigurator configurator = FileTQEConfigurator.tqe3Builder( IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(SIMPLE_GRPC_CONFIG)) @@ -109,17 +70,44 @@ void testPublishAndConsumeData() { final String queueName = "test"; - final InetSocketAddress producerAddress = findGrpcAddress(cluster, GrpcRole.PRODUCER); - final InetSocketAddress consumerAddress = findGrpcAddress(cluster, GrpcRole.CONSUMER); + final List> producers = + cluster.grpc().values().stream() + .filter(g -> g.roles().contains(GrpcRole.PRODUCER)) + .toList(); + final List> consumers = + cluster.grpc().values().stream() + .filter(g -> g.roles().contains(GrpcRole.CONSUMER)) + .toList(); + + Assertions.assertFalse(producers.isEmpty()); + Assertions.assertFalse(consumers.isEmpty()); + + final Set grpcAddresses = producers.get(0).grpcAddresses(); + final Set consumerAddresses = consumers.get(0).grpcAddresses(); - final ManagedChannel producerChannel = createReadyChannel(producerAddress); - final ManagedChannel consumerChannel = createReadyChannel(consumerAddress); + final Optional producerAddress = grpcAddresses.stream().findFirst(); + Assertions.assertTrue(producerAddress.isPresent()); + final Optional consumerAddress = + consumerAddresses.stream().findFirst(); + Assertions.assertTrue(consumerAddress.isPresent()); - final ProducerBlockingStub producer = ProducerGrpc.newBlockingStub(producerChannel); - final ConsumerServiceStub consumer = ConsumerServiceGrpc.newStub(consumerChannel); + final ManagedChannel producerChannel = + TQETestHelper.createReadyChannel(producerAddress.get()); + final ManagedChannel consumerChannel = + TQETestHelper.createReadyChannel(consumerAddress.get()); try { - final List users = generateTestUsers(); + final ProducerBlockingStub producer = ProducerGrpc.newBlockingStub(producerChannel); + final ConsumerServiceStub consumer = ConsumerServiceGrpc.newStub(consumerChannel); + + final List users = + Instancio.ofList(User.class) + .size(100) + .generate( + Select.field(User::getName), + g -> g.string().alphaNumeric().allowEmpty().nullable()) + .generate(Select.field(User::getAge), Generators::ints) + .create(); Unreliables.retryUntilSuccess( 60, @@ -129,7 +117,8 @@ void testPublishAndConsumeData() { ProduceRequest.newBuilder().setQueue(queueName); for (User user : users) { requestBuilder.addMessages( - ProduceMessage.newBuilder().setPayload(serializeUser(user))); + ProduceMessage.newBuilder() + .setPayload(ByteString.copyFrom(MAPPER.writeValueAsBytes(user)))); } producer.produce(requestBuilder.build()); return true; @@ -146,7 +135,16 @@ void testPublishAndConsumeData() { @Override public void onNext(SubscriptionStreamResponse response) { response.getNotifications().getNotificationsList().stream() - .map(n -> deserializeUser(n.getMessage().getPayload())) + .map( + n -> { + try { + return MAPPER.readValue( + n.getMessage().getPayload().toByteArray(), + User.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) .forEach(result::add); } @@ -166,7 +164,9 @@ public void onCompleted() {} return true; }); - assertAllUsersConsumed(users, result); + Unreliables.retryUntilTrue( + 60, TimeUnit.SECONDS, () -> new LinkedHashSet<>(users).size() == result.size()); + Assertions.assertEquals(new LinkedHashSet<>(users), result); } finally { consumerChannel.shutdownNow(); producerChannel.shutdownNow(); diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterTest.java new file mode 100644 index 00000000..bc7a6b55 --- /dev/null +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterTest.java @@ -0,0 +1,351 @@ +/* + * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY + * All Rights Reserved. + */ + +package org.testcontainers.containers.integration.tqe; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.testcontainers.containers.ContainerLaunchException; +import org.testcontainers.containers.tqe.TQECluster; +import org.testcontainers.containers.tqe.configuration.TQEConfigurator; + +class TQEClusterTest { + + @ParameterizedTest + @MethodSource("tqeVersions") + void testMultiplyRestart(TQETestHelper.TQETestParams p) throws Exception { + try (TQEConfigurator configurator = + p.configuratorBuilderFactory().apply(p.queueConfig(), Set.of(p.grpcConfig())).build(); + TQECluster cluster = p.clusterFactory().apply(configurator)) { + cluster.start(); + } + } + + @Test + void testRestartMethod() throws Exception { + for (TQETestHelper.TQETestParams p : TQETestHelper.tqeVersions().toList()) { + try (TQEConfigurator configurator = + p.configuratorBuilderFactory() + .apply(p.queueConfig(), Set.of(p.grpcConfig())) + .build(); + TQECluster cluster = p.clusterFactory().apply(configurator)) { + cluster.start(); + cluster.restart(1, TimeUnit.SECONDS, 1, TimeUnit.SECONDS); + } + } + } + + public static Stream dataForTestInvalidQueueConfigShouldThrow() { + final List invalidConfigs = + Arrays.asList( + // no required test-super user + """ + # Credentials + credentials: + users: + admin: + password: 'secret-cluster-cookie' + roles: [ super ] + replicator: + password: 'secret' + roles: [ replication ] + storage: + roles: [ sharding ] + password: storage + + # advertise configs for all nodes + iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + password: storage + + roles: [ roles.metrics-export ] + # queues configs + roles_cfg: + app.roles.queue: + queues: + - name: test + deduplication_mode: keep_latest + disabled_filters_by: [ sharding_key ] + roles.metrics-export: + http: + - listen: 8081 + endpoints: + - format: prometheus + path: '/metrics' + + groups: + routers: + replicasets: + r-1: + sharding: + roles: [ router ] + roles: [ app.roles.api ] + instances: + router: + iproto: + listen: + - uri: router:3301 + storages: + replicasets: + shard-1: + replication: + failover: manual + sharding: + roles: [ storage ] + leader: master + instances: + master: + iproto: + listen: + - uri: master:3301 + net_msg_max: 768 + """, + // no consumer storage to connect from grpc + """ + # Credentials + credentials: + users: + test-super: + password: 'test' + roles: [ super ] + admin: + password: 'secret-cluster-cookie' + roles: [ super ] + replicator: + password: 'secret' + roles: [ replication ] + storage: + roles: [ sharding ] + password: storage + + # advertise configs for all nodes + iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + password: storage + + roles: [ roles.metrics-export ] + # queues configs + roles_cfg: + app.roles.queue: + queues: + - name: test + deduplication_mode: keep_latest + disabled_filters_by: [ sharding_key ] + roles.metrics-export: + http: + - listen: 8081 + endpoints: + - format: prometheus + path: '/metrics' + + groups: + routers: + replicasets: + r-1: + sharding: + roles: [ router ] + roles: [ app.roles.api ] + instances: + router: + iproto: + listen: + - uri: router:3301 + """); + + return invalidConfigs.stream() + .map( + s -> { + final Path testConfigPath = + TQETestHelper.TEST_TEMP_DIR.resolve(UUID.randomUUID().toString()); + try { + Files.writeString(testConfigPath, s); + return Arguments.of(testConfigPath); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + @ParameterizedTest + @MethodSource("dataForTestInvalidQueueConfigShouldThrow") + void testInvalidQueueConfigShouldThrow(Path queueConfig) { + for (TQETestHelper.TQETestParams p : TQETestHelper.tqeVersions().toList()) { + Assertions.assertThrows( + ContainerLaunchException.class, + () -> { + try (TQEConfigurator configurator = + p.configuratorBuilderFactory() + .apply(queueConfig, Set.of(p.grpcConfig())) + .withStartupTimeout(Duration.ofSeconds(5)) + .build(); + TQECluster cluster = p.clusterFactory().apply(configurator)) { + cluster.start(); + } + }); + } + } + + public static Stream dataForTestInvalidGrpcConfig() { + return TQETestHelper.tqeVersions() + .flatMap( + p -> { + final List invalidGrpcConfigs = + Arrays.asList( + // unknown host + """ + core_port: 1111 + grpc_listen: + - uri: 'tcp://0.0.0.0:18182' + + %s: + enabled: true + tarantool: + user: test-super + pass: test + connections: + routers: + - "unknown:3301" + + consumer: + enabled: true + tarantool: + user: test-super + pass: test + connections: + storage: + - "master:3301" + """ + .formatted(p.producerRoleName()), + // no consumers and producers + """ + core_port: 1111 + grpc_listen: + - uri: 'tcp://0.0.0.0:18182' + + %s: + enabled: false + tarantool: + user: test-super + pass: test + connections: + routers: + - "router:3301" + + consumer: + enabled: false + tarantool: + user: test-super + pass: test + connections: + storage: + - "master:3301" + """ + .formatted(p.producerRoleName()), + // no core_port parameter + """ + grpc_listen: + - uri: 'tcp://0.0.0.0:18182' + + %s: + enabled: true + tarantool: + user: test-super + pass: test + connections: + routers: + - "router:3301" + + consumer: + enabled: true + tarantool: + user: test-super + pass: test + connections: + storage: + - "master:3301" + """ + .formatted(p.producerRoleName()), + // no listen.uri parameter + """ + core_port: 1111 + + %s: + enabled: true + tarantool: + user: test-super + pass: test + connections: + routers: + - "router:3301" + + consumer: + enabled: true + tarantool: + user: test-super + pass: test + connections: + storage: + - "master:3301" + """ + .formatted(p.producerRoleName())); + + return invalidGrpcConfigs.stream() + .map( + s -> { + final Path testConfigPath = + TQETestHelper.TEST_TEMP_DIR.resolve(UUID.randomUUID() + ".yml"); + try { + Files.writeString(testConfigPath, s); + return Arguments.of(p, testConfigPath); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + }); + } + + @ParameterizedTest + @MethodSource("dataForTestInvalidGrpcConfig") + void testInvalidGrpcConfig(TQETestHelper.TQETestParams p, Path grpcConfig) { + Assertions.assertThrows( + ContainerLaunchException.class, + () -> { + try (TQEConfigurator configurator = + p.configuratorBuilderFactory() + .apply(p.queueConfig(), Set.of(grpcConfig)) + .withStartupTimeout(Duration.ofSeconds(5)) + .build(); + TQECluster cluster = p.clusterFactory().apply(configurator)) { + cluster.start(); + } + }); + } + + static Stream tqeVersions() { + return TQETestHelper.tqeVersions(); + } +} diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQETestHelper.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQETestHelper.java new file mode 100644 index 00000000..e412505e --- /dev/null +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQETestHelper.java @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY + * All Rights Reserved. + */ + +package org.testcontainers.containers.integration.tqe; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Stream; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import org.rnorth.ducttape.unreliables.Unreliables; +import org.testcontainers.containers.tqe.TQE2ClusterImpl; +import org.testcontainers.containers.tqe.TQE3ClusterImpl; +import org.testcontainers.containers.tqe.TQECluster; +import org.testcontainers.containers.tqe.configuration.FileTQEConfigurator; +import org.testcontainers.containers.tqe.configuration.TQEConfigurator; +import org.testcontainers.utility.DockerImageName; + +final class TQETestHelper { + + static final Path TEST_TEMP_DIR; + + static { + try { + TEST_TEMP_DIR = Files.createTempDirectory("tqe-test-"); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static final DockerImageName IMAGE_NAME_2 = + DockerImageName.parse( + System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") + + "tarantool/message-queue-ee:2.5.3"); + + private static final DockerImageName IMAGE_NAME_3 = + DockerImageName.parse( + System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") + + "tarantool/message-queue-ee:v3.5.0"); + + private static final Path GRPC_CONFIG_2 = loadConfig("tqe2/simple-config/simple-grpc.yml"); + private static final Path QUEUE_CONFIG_2 = loadConfig("tqe2/simple-config/simple-queue.yml"); + private static final Path GRPC_CONFIG_3 = loadConfig("tqe3/simple-config/simple-grpc.yml"); + private static final Path QUEUE_CONFIG_3 = loadConfig("tqe3/simple-config/simple-queue.yml"); + + private TQETestHelper() {} + + static Path loadConfig(String resourcePath) { + try { + return Paths.get( + Objects.requireNonNull(TQETestHelper.class.getClassLoader().getResource(resourcePath)) + .toURI()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + static ManagedChannel createReadyChannel(InetSocketAddress address) { + return Unreliables.retryUntilSuccess( + 60, + TimeUnit.SECONDS, + () -> { + ManagedChannel ch = + ManagedChannelBuilder.forAddress(address.getHostName(), address.getPort()) + .usePlaintext() + .maxInboundMessageSize(16 * 1024 * 1024) + .keepAliveTime(30, TimeUnit.SECONDS) + .keepAliveTimeout(5, TimeUnit.SECONDS) + .keepAliveWithoutCalls(true) + .build(); + + ch.getState(true); + Unreliables.retryUntilTrue( + 5, + TimeUnit.SECONDS, + () -> { + io.grpc.ConnectivityState state = ch.getState(false); + if (state == io.grpc.ConnectivityState.READY) { + return true; + } + ch.resetConnectBackoff(); + Thread.sleep(100); + return false; + }); + return ch; + }); + } + + static Stream tqeVersions() { + return Stream.of( + new TQETestParams( + "TQE2", + IMAGE_NAME_2, + QUEUE_CONFIG_2, + GRPC_CONFIG_2, + "publisher", + (qc, gc) -> FileTQEConfigurator.tqe2Builder(IMAGE_NAME_2, qc, gc), + TQE2ClusterImpl::new, + false), + new TQETestParams( + "TQE3", + IMAGE_NAME_3, + QUEUE_CONFIG_3, + GRPC_CONFIG_3, + "producer", + (qc, gc) -> FileTQEConfigurator.tqe3Builder(IMAGE_NAME_3, qc, gc), + TQE3ClusterImpl::new, + true)); + } + + record TQETestParams( + String displayName, + DockerImageName imageName, + Path queueConfig, + Path grpcConfig, + String producerRoleName, + BiFunction, FileTQEConfigurator.Builder> configuratorBuilderFactory, + Function clusterFactory, + boolean requiresConfigure) { + + @Override + public String toString() { + return displayName; + } + } +} diff --git a/testcontainers/src/test/resources/tqe2/invalid-config/no-consumer-storage.yml b/testcontainers/src/test/resources/tqe2/invalid-config/no-consumer-storage.yml deleted file mode 100644 index 0732bbb2..00000000 --- a/testcontainers/src/test/resources/tqe2/invalid-config/no-consumer-storage.yml +++ /dev/null @@ -1,52 +0,0 @@ -# Credentials -credentials: - users: - test-super: - password: 'test' - roles: [ super ] - admin: - password: 'secret-cluster-cookie' - roles: [ super ] - replicator: - password: 'secret' - roles: [ replication ] - storage: - roles: [ sharding ] - password: storage - -# advertise configs for all nodes -iproto: - advertise: - peer: - login: replicator - sharding: - login: storage - password: storage - -roles: [ roles.metrics-export ] -# queues configs -roles_cfg: - app.roles.queue: - queues: - - name: test - deduplication_mode: keep_latest - disabled_filters_by: [ sharding_key ] - roles.metrics-export: - http: - - listen: 8081 - endpoints: - - format: prometheus - path: '/metrics' - -groups: - routers: - replicasets: - r-1: - sharding: - roles: [ router ] - roles: [ app.roles.api ] - instances: - router: - iproto: - listen: - - uri: router:3301 diff --git a/testcontainers/src/test/resources/tqe2/invalid-config/no-test-super-user.yml b/testcontainers/src/test/resources/tqe2/invalid-config/no-test-super-user.yml deleted file mode 100644 index 2bc57ef7..00000000 --- a/testcontainers/src/test/resources/tqe2/invalid-config/no-test-super-user.yml +++ /dev/null @@ -1,63 +0,0 @@ -# Credentials -credentials: - users: - admin: - password: 'secret-cluster-cookie' - roles: [ super ] - replicator: - password: 'secret' - roles: [ replication ] - storage: - roles: [ sharding ] - password: storage - -# advertise configs for all nodes -iproto: - advertise: - peer: - login: replicator - sharding: - login: storage - password: storage - -roles: [ roles.metrics-export ] -# queues configs -roles_cfg: - app.roles.queue: - queues: - - name: test - deduplication_mode: keep_latest - disabled_filters_by: [ sharding_key ] - roles.metrics-export: - http: - - listen: 8081 - endpoints: - - format: prometheus - path: '/metrics' - -groups: - routers: - replicasets: - r-1: - sharding: - roles: [ router ] - roles: [ app.roles.api ] - instances: - router: - iproto: - listen: - - uri: router:3301 - storages: - replicasets: - shard-1: - replication: - failover: manual - sharding: - roles: [ storage ] - leader: master - instances: - master: - iproto: - listen: - - uri: master:3301 - net_msg_max: 768 diff --git a/testcontainers/src/test/resources/tqe2/invalid-config/router-no-required-roles.yml b/testcontainers/src/test/resources/tqe2/invalid-config/router-no-required-roles.yml deleted file mode 100644 index 1614cc27..00000000 --- a/testcontainers/src/test/resources/tqe2/invalid-config/router-no-required-roles.yml +++ /dev/null @@ -1,61 +0,0 @@ -# Credentials -credentials: - users: - test-super: - password: 'test' - roles: [ super ] - admin: - password: 'secret-cluster-cookie' - roles: [ super ] - replicator: - password: 'secret' - roles: [ replication ] - storage: - roles: [ sharding ] - password: storage -# advertise configs for all nodes -iproto: - advertise: - peer: - login: replicator - sharding: - login: storage - password: storage -roles: [ roles.metrics-export ] -# queues configs -roles_cfg: - app.roles.queue: - queues: - - name: test - deduplication_mode: keep_latest - disabled_filters_by: [ sharding_key ] - roles.metrics-export: - http: - - listen: 8081 - endpoints: - - format: prometheus - path: '/metrics' -groups: - routers: - replicasets: - r-1: - sharding: - roles: [ router ] - instances: - router: - iproto: - listen: - - uri: router:3301 - storages: - replicasets: - shard-1: - replication: - failover: manual - sharding: - roles: [ storage ] - leader: master - instances: - master: - iproto: - listen: - - uri: master:3301 diff --git a/testcontainers/src/test/resources/tqe2/invalid-config/storage-no-listen-uri.yml b/testcontainers/src/test/resources/tqe2/invalid-config/storage-no-listen-uri.yml deleted file mode 100644 index dd9b9475..00000000 --- a/testcontainers/src/test/resources/tqe2/invalid-config/storage-no-listen-uri.yml +++ /dev/null @@ -1,61 +0,0 @@ -# Credentials -credentials: - users: - test-super: - password: 'test' - roles: [ super ] - admin: - password: 'secret-cluster-cookie' - roles: [ super ] - replicator: - password: 'secret' - roles: [ replication ] - storage: - roles: [ sharding ] - password: storage -# advertise configs for all nodes -iproto: - advertise: - peer: - login: replicator - sharding: - login: storage - password: storage -roles: [ roles.metrics-export ] -# queues configs -roles_cfg: - app.roles.queue: - queues: - - name: test - deduplication_mode: keep_latest - disabled_filters_by: [ sharding_key ] - roles.metrics-export: - http: - - listen: 8081 - endpoints: - - format: prometheus - path: '/metrics' -groups: - routers: - replicasets: - r-1: - sharding: - roles: [ router ] - roles: [ app.roles.api ] - instances: - router: - iproto: - listen: - - uri: router:3301 - storages: - replicasets: - shard-1: - replication: - failover: manual - sharding: - roles: [ storage ] - leader: master - instances: - master: - iproto: - net_msg_max: 768 diff --git a/testcontainers/src/test/resources/tqe3/invalid-config/no-consumer-storage.yml b/testcontainers/src/test/resources/tqe3/invalid-config/no-consumer-storage.yml deleted file mode 100644 index 1185efe0..00000000 --- a/testcontainers/src/test/resources/tqe3/invalid-config/no-consumer-storage.yml +++ /dev/null @@ -1,53 +0,0 @@ -# Credentials -credentials: - users: - test-super: - password: 'test' - roles: [ super ] - admin: - password: 'secret-cluster-cookie' - roles: [ super ] - replicator: - password: 'secret' - roles: [ replication ] - storage: - roles: [ sharding ] - password: storage - -# advertise configs for all nodes -iproto: - advertise: - peer: - login: replicator - sharding: - login: storage - password: storage - -roles: [ roles.metrics-export ] -# queues configs -roles_cfg: - roles.tqe-storage: - queues: - - name: test - deduplication_mode: keep_latest - disabled_filters_by: [ sharding_key ] - roles.metrics-export: - http: - - listen: 8081 - endpoints: - - format: prometheus - path: '/metrics' - -groups: - routers: - replicasets: - router-1: - sharding: - roles: [ router ] - roles: - - roles.tqe-router - instances: - router: - iproto: - listen: - - uri: router:3301 diff --git a/testcontainers/src/test/resources/tqe3/invalid-config/no-test-super-user.yml b/testcontainers/src/test/resources/tqe3/invalid-config/no-test-super-user.yml deleted file mode 100644 index 87b9e9dd..00000000 --- a/testcontainers/src/test/resources/tqe3/invalid-config/no-test-super-user.yml +++ /dev/null @@ -1,66 +0,0 @@ -# Credentials -credentials: - users: - admin: - password: 'secret-cluster-cookie' - roles: [ super ] - replicator: - password: 'secret' - roles: [ replication ] - storage: - roles: [ sharding ] - password: storage - -# advertise configs for all nodes -iproto: - advertise: - peer: - login: replicator - sharding: - login: storage - password: storage - -roles: [ roles.metrics-export ] -# queues configs -roles_cfg: - roles.tqe-storage: - queues: - - name: test - deduplication_mode: keep_latest - disabled_filters_by: [ sharding_key ] - roles.metrics-export: - http: - - listen: 8081 - endpoints: - - format: prometheus - path: '/metrics' - -groups: - routers: - replicasets: - router-1: - sharding: - roles: [ router ] - roles: - - roles.tqe-router - instances: - router: - iproto: - listen: - - uri: router:3301 - storages: - replicasets: - storage-1: - replication: - failover: manual - sharding: - roles: [ storage ] - roles: - - roles.tqe-storage - leader: master - instances: - master: - iproto: - listen: - - uri: master:3301 - net_msg_max: 768 diff --git a/testcontainers/src/test/resources/tqe3/invalid-config/router-no-required-roles.yml b/testcontainers/src/test/resources/tqe3/invalid-config/router-no-required-roles.yml deleted file mode 100644 index ae9818ac..00000000 --- a/testcontainers/src/test/resources/tqe3/invalid-config/router-no-required-roles.yml +++ /dev/null @@ -1,63 +0,0 @@ -# Credentials -credentials: - users: - test-super: - password: 'test' - roles: [ super ] - admin: - password: 'secret-cluster-cookie' - roles: [ super ] - replicator: - password: 'secret' - roles: [ replication ] - storage: - roles: [ sharding ] - password: storage -# advertise configs for all nodes -iproto: - advertise: - peer: - login: replicator - sharding: - login: storage - password: storage -roles: [ roles.metrics-export ] -# queues configs -roles_cfg: - roles.tqe-storage: - queues: - - name: test - deduplication_mode: keep_latest - disabled_filters_by: [ sharding_key ] - roles.metrics-export: - http: - - listen: 8081 - endpoints: - - format: prometheus - path: '/metrics' -groups: - routers: - replicasets: - router-1: - sharding: - roles: [ router ] - instances: - router: - iproto: - listen: - - uri: router:3301 - storages: - replicasets: - storage-1: - replication: - failover: manual - sharding: - roles: [ storage ] - roles: - - roles.tqe-storage - leader: master - instances: - master: - iproto: - listen: - - uri: master:3301 diff --git a/testcontainers/src/test/resources/tqe3/invalid-config/storage-no-listen-uri.yml b/testcontainers/src/test/resources/tqe3/invalid-config/storage-no-listen-uri.yml deleted file mode 100644 index 49033720..00000000 --- a/testcontainers/src/test/resources/tqe3/invalid-config/storage-no-listen-uri.yml +++ /dev/null @@ -1,62 +0,0 @@ -# Credentials -credentials: - users: - test-super: - password: 'test' - roles: [ super ] - admin: - password: 'secret-cluster-cookie' - roles: [ super ] - replicator: - password: 'secret' - roles: [ replication ] - storage: - roles: [ sharding ] - password: storage -# advertise configs for all nodes -iproto: - advertise: - peer: - login: replicator - sharding: - login: storage - password: storage -roles: [ roles.metrics-export ] -# queues configs -roles_cfg: - roles.tqe-storage: - queues: - - name: test - deduplication_mode: keep_latest - disabled_filters_by: [ sharding_key ] - roles.metrics-export: - http: - - listen: 8081 - endpoints: - - format: prometheus - path: '/metrics' -groups: - routers: - replicasets: - router-1: - sharding: - roles: [ router ] - roles: - - roles.tqe-router - instances: - router: - iproto: - listen: - - uri: router:3301 - storages: - replicasets: - storage-1: - replication: - failover: manual - sharding: - roles: [ storage ] - leader: master - instances: - master: - iproto: - net_msg_max: 768 From c873d046e04faf517400d6318d82ab8da8cd1ccd Mon Sep 17 00:00:00 2001 From: Dmitry Kasimovskiy Date: Fri, 5 Jun 2026 16:03:14 +0300 Subject: [PATCH 5/5] refactor(testcontainers): consolidate TQE test classes into parameterized suite Replace TQE2ClusterImplTest + TQE3ClusterImplTest (~80% duplication) with a single TQEClusterIntegrationTest driven by a TQEVersion enum and GrpcTestStrategy. Encapsulate cluster lifecycle in TQEClusterFixture. - Add TQEVersion enum: encapsulates image, configs, role names, builder/ cluster factories, gRPC strategy per version (OCP-friendly). - Add GrpcTestStrategy interface + TQE2/TQE3 implementations: TQE2 uses PublisherServiceGrpc.publishBatch + unidirectional subscribe, TQE3 uses ProducerGrpc.produce + bidirectional subscribe. - Add TQEClusterFixture (AutoCloseable) for cluster lifecycle. - Parameterize TQEClusterTest (testRestartMethod) and FileTQEConfiguratorTest (testInvalidQueueConfig) via @EnumSource. - Simplify TQETestHelper: remove TQETestParams record, tqeVersions() stream, and duplicate image/config constants (moved to TQEVersion). - Fix GrpcContainerImpl.validateConfigPath: was missing the ! and used Path.endsWith(String) (path-component comparison) instead of String.endsWith substring check. Now uses path.toString().endsWith() so valid .yml configs pass and non-yml files are rejected as the error message claims. - Delete TQE2ClusterImplTest and TQE3ClusterImplTest (348 lines removed). Verified: 32/32 TQE tests pass via ./mvnw -pl testcontainers -Dtest='*TQE*' test. Co-Authored-By: Claude Opus 4.7 --- .../containers/tqe/GrpcContainerImpl.java | 2 +- .../tqe/FileTQEConfiguratorTest.java | 49 +++-- .../integration/tqe/GrpcTestStrategy.java | 32 ++++ .../integration/tqe/TQE2ClusterImplTest.java | 171 ----------------- .../integration/tqe/TQE2GrpcTestStrategy.java | 75 ++++++++ .../integration/tqe/TQE3ClusterImplTest.java | 177 ------------------ .../integration/tqe/TQE3GrpcTestStrategy.java | 80 ++++++++ .../integration/tqe/TQEClusterFixture.java | 80 ++++++++ .../tqe/TQEClusterIntegrationTest.java | 81 ++++++++ .../integration/tqe/TQEClusterTest.java | 113 +++++------ .../integration/tqe/TQETestHelper.java | 63 ------- .../integration/tqe/TQEVersion.java | 164 ++++++++++++++++ 12 files changed, 586 insertions(+), 501 deletions(-) create mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/GrpcTestStrategy.java delete mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java create mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2GrpcTestStrategy.java delete mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java create mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3GrpcTestStrategy.java create mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterFixture.java create mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterIntegrationTest.java create mode 100644 testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEVersion.java diff --git a/testcontainers/src/main/java/org/testcontainers/containers/tqe/GrpcContainerImpl.java b/testcontainers/src/main/java/org/testcontainers/containers/tqe/GrpcContainerImpl.java index ccd15aba..b20ce2a4 100644 --- a/testcontainers/src/main/java/org/testcontainers/containers/tqe/GrpcContainerImpl.java +++ b/testcontainers/src/main/java/org/testcontainers/containers/tqe/GrpcContainerImpl.java @@ -163,7 +163,7 @@ private static void validateConfigPath(Path configPath) { if (configPath == null || !Files.exists(configPath) || !Files.isRegularFile(configPath) - || configPath.endsWith(".yml")) { + || !configPath.toString().endsWith(".yml")) { LOGGER.error( "Invalid config file. Config path is null or not exists or not regular or not having" + " '.yml' extension: {}", diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQEConfiguratorTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQEConfiguratorTest.java index 1d001d4b..1cbaab55 100644 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQEConfiguratorTest.java +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/FileTQEConfiguratorTest.java @@ -15,6 +15,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import org.testcontainers.containers.ContainerLaunchException; import org.testcontainers.containers.tqe.configuration.FileTQEConfigurator; @@ -24,12 +25,12 @@ class FileTQEConfiguratorTest { @ParameterizedTest - @MethodSource("tqeVersions") - void simpleConfiguration(TQETestHelper.TQETestParams p) throws Exception { + @EnumSource(TQEVersion.class) + void simpleConfiguration(TQEVersion version) throws Exception { try (TQEConfigurator configurator = - p.configuratorBuilderFactory().apply(p.queueConfig(), Set.of(p.grpcConfig())).build()) { + version.configuratorBuilder(version.queueConfig(), Set.of(version.grpcConfig())).build()) { configurator.queue().values().parallelStream().forEach(Startable::start); - if (p.requiresConfigure()) { + if (version.requiresConfigure()) { configurator.configure(); } configurator.grpc().values().parallelStream().forEach(Startable::start); @@ -170,57 +171,51 @@ public static Stream dataForTestInvalidQueueConfig() { @ParameterizedTest @MethodSource("dataForTestInvalidQueueConfig") void testInvalidQueueConfig(String invalidQueueConfig) throws IOException { - for (TQETestHelper.TQETestParams p : TQETestHelper.tqeVersions().toList()) { - final Path invalidConfigPath = - TQETestHelper.TEST_TEMP_DIR.resolve(UUID.randomUUID().toString()); - Files.writeString(invalidConfigPath, invalidQueueConfig); + final Path invalidConfigPath = + TQETestHelper.TEST_TEMP_DIR.resolve(UUID.randomUUID().toString()); + Files.writeString(invalidConfigPath, invalidQueueConfig); + for (TQEVersion version : TQEVersion.values()) { Assertions.assertThrows( ContainerLaunchException.class, () -> { try (FileTQEConfigurator c = - p.configuratorBuilderFactory() - .apply(invalidConfigPath, Set.of(p.grpcConfig())) + version + .configuratorBuilder(invalidConfigPath, Set.of(version.grpcConfig())) .build()) {} }); } } public static Stream dataForTestInvalidConfigsPaths() { - return TQETestHelper.tqeVersions() + return TQEVersion.all() .flatMap( - (TQETestHelper.TQETestParams p) -> { - Path qc = p.queueConfig(); - Path gc = p.grpcConfig(); + version -> { + Path qc = version.queueConfig(); + Path gc = version.grpcConfig(); return Stream.of( // invalid grpc configs // null - Arguments.of(p, qc, null), + Arguments.of(version, qc, null), // empty - Arguments.of(p, qc, Set.of()), + Arguments.of(version, qc, Set.of()), // non regular - Arguments.of(p, qc, Set.of(TQETestHelper.TEST_TEMP_DIR)), + Arguments.of(version, qc, Set.of(TQETestHelper.TEST_TEMP_DIR)), // invalid queue config - Arguments.of(p, (Path) null, Set.of(gc)), - Arguments.of(p, TQETestHelper.TEST_TEMP_DIR, Set.of(gc))); + Arguments.of(version, (Path) null, Set.of(gc)), + Arguments.of(version, TQETestHelper.TEST_TEMP_DIR, Set.of(gc))); }); } @ParameterizedTest @MethodSource("dataForTestInvalidConfigsPaths") void testInvalidConfigsPaths( - TQETestHelper.TQETestParams p, Path invalidGrpcConfig, Set invalidQueueConfigs) { + TQEVersion version, Path invalidGrpcConfig, Set invalidQueueConfigs) { Assertions.assertThrows( IllegalArgumentException.class, () -> { try (FileTQEConfigurator c = - p.configuratorBuilderFactory() - .apply(invalidGrpcConfig, invalidQueueConfigs) - .build()) {} + version.configuratorBuilder(invalidGrpcConfig, invalidQueueConfigs).build()) {} }); } - - static Stream tqeVersions() { - return TQETestHelper.tqeVersions(); - } } diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/GrpcTestStrategy.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/GrpcTestStrategy.java new file mode 100644 index 00000000..60585cc7 --- /dev/null +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/GrpcTestStrategy.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY + * All Rights Reserved. + */ + +package org.testcontainers.containers.integration.tqe; + +import java.util.List; +import java.util.Set; + +import io.grpc.ManagedChannel; +import org.testcontainers.containers.utils.pojo.User; + +/** + * Encapsulates version-specific gRPC API for publishing and subscribing. TQE 2.x uses {@code + * PublisherServiceGrpc} + unidirectional streaming. TQE 3.x uses {@code ProducerGrpc} + + * bidirectional streaming. + */ +interface GrpcTestStrategy { + + /** + * Publishes a batch of users to the given queue over the channel. Synchronous: throws on failure. + */ + void publish(ManagedChannel channel, List users, String queue) throws Exception; + + /** + * Starts a subscription on the given queue. Messages are delivered asynchronously and added to + * {@code result} as they arrive. This call only kicks off the subscription; callers should + * retry/poll until {@code result} reaches the expected size. + */ + void subscribe(ManagedChannel channel, String queue, Set result); +} diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java deleted file mode 100644 index 526a08e2..00000000 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2ClusterImplTest.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY - * All Rights Reserved. - */ - -package org.testcontainers.containers.integration.tqe; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.file.Path; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArraySet; -import java.util.concurrent.TimeUnit; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.protobuf.ByteString; -import io.grpc.ManagedChannel; -import io.grpc.stub.StreamObserver; -import org.instancio.Instancio; -import org.instancio.Select; -import org.instancio.generators.Generators; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.RepeatedTest; -import org.rnorth.ducttape.unreliables.Unreliables; -import org.testcontainers.containers.tqe.GrpcContainer; -import org.testcontainers.containers.tqe.GrpcContainer.GrpcRole; -import org.testcontainers.containers.tqe.TQE2ClusterImpl; -import org.testcontainers.containers.tqe.TQECluster; -import org.testcontainers.containers.tqe.configuration.FileTQEConfigurator; -import org.testcontainers.containers.tqe.configuration.TQEConfigurator; -import org.testcontainers.containers.utils.pojo.User; -import org.testcontainers.utility.DockerImageName; -import tarantool.queue_ee.v2.Consumer.SubscriptionNotifications; -import tarantool.queue_ee.v2.Consumer.SubscriptionRequest; -import tarantool.queue_ee.v2.ConsumerServiceGrpc; -import tarantool.queue_ee.v2.ConsumerServiceGrpc.ConsumerServiceStub; -import tarantool.queue_ee.v2.Publisher.BatchRequestMessage; -import tarantool.queue_ee.v2.Publisher.PublishBatchRequest; -import tarantool.queue_ee.v2.PublisherServiceGrpc; -import tarantool.queue_ee.v2.PublisherServiceGrpc.PublisherServiceBlockingStub; - -class TQE2ClusterImplTest { - - private static final DockerImageName IMAGE_NAME = - DockerImageName.parse( - System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") - + "tarantool/message-queue-ee:2.5.3"); - - private static final Path SIMPLE_GRPC_CONFIG = - TQETestHelper.loadConfig("tqe2/simple-config/simple-grpc.yml"); - private static final Path SIMPLE_QUEUE_CONFIG = - TQETestHelper.loadConfig("tqe2/simple-config/simple-queue.yml"); - - @RepeatedTest(10) - void testPublishAndConsumeData() { - Assertions.assertDoesNotThrow( - () -> { - final ObjectMapper MAPPER = new ObjectMapper(); - - try (TQEConfigurator configurator = - FileTQEConfigurator.tqe2Builder( - IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(SIMPLE_GRPC_CONFIG)) - .build(); - TQECluster cluster = new TQE2ClusterImpl(configurator)) { - cluster.start(); - - final String queueName = "test"; - - final List> publishers = - cluster.grpc().values().stream() - .filter(g -> g.roles().contains(GrpcRole.PUBLISHER)) - .toList(); - final List> consumers = - cluster.grpc().values().stream() - .filter(g -> g.roles().contains(GrpcRole.CONSUMER)) - .toList(); - - Assertions.assertFalse(publishers.isEmpty()); - Assertions.assertFalse(consumers.isEmpty()); - - final Set grpcAddresses = publishers.get(0).grpcAddresses(); - final Set consumerAddresses = consumers.get(0).grpcAddresses(); - - final Optional publisherAddress = grpcAddresses.stream().findFirst(); - Assertions.assertTrue(publisherAddress.isPresent()); - final Optional consumerAddress = - consumerAddresses.stream().findFirst(); - Assertions.assertTrue(consumerAddress.isPresent()); - - final ManagedChannel publisherChannel = - TQETestHelper.createReadyChannel(publisherAddress.get()); - final ManagedChannel consumerChannel = - TQETestHelper.createReadyChannel(consumerAddress.get()); - - try { - final PublisherServiceBlockingStub pService = - PublisherServiceGrpc.newBlockingStub(publisherChannel); - final ConsumerServiceStub cService = ConsumerServiceGrpc.newStub(consumerChannel); - - final List users = - Instancio.ofList(User.class) - .size(100) - .generate( - Select.field(User::getName), - g -> g.string().alphaNumeric().allowEmpty().nullable()) - .generate(Select.field(User::getAge), Generators::ints) - .create(); - - Unreliables.retryUntilSuccess( - 60, - TimeUnit.SECONDS, - () -> { - final PublishBatchRequest.Builder requestBuilder = - PublishBatchRequest.newBuilder(); - for (User user : users) { - requestBuilder.addMessages( - BatchRequestMessage.newBuilder() - .setPayload(ByteString.copyFrom(MAPPER.writeValueAsBytes(user)))); - } - pService.publishBatch(requestBuilder.setQueue(queueName).build()); - return true; - }); - - final Set result = new CopyOnWriteArraySet<>(); - Unreliables.retryUntilSuccess( - 60, - TimeUnit.SECONDS, - () -> { - cService.subscribe( - SubscriptionRequest.newBuilder().setCursor("").setQueue(queueName).build(), - new StreamObserver<>() { - @Override - public void onNext(SubscriptionNotifications value) { - value.getNotificationsList().stream() - .map( - n -> { - try { - return MAPPER.readValue( - n.getMessage().getPayload().toByteArray(), User.class); - } catch (IOException e) { - throw new RuntimeException(e); - } - }) - .forEach(result::add); - } - - @Override - public void onError(Throwable t) { - throw new RuntimeException("Stream observer received error", t); - } - - @Override - public void onCompleted() {} - }); - return true; - }); - - Unreliables.retryUntilTrue( - 60, TimeUnit.SECONDS, () -> new LinkedHashSet<>(users).size() == result.size()); - Assertions.assertEquals(new LinkedHashSet<>(users), result); - } finally { - consumerChannel.shutdownNow(); - publisherChannel.shutdownNow(); - } - } - }); - } -} diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2GrpcTestStrategy.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2GrpcTestStrategy.java new file mode 100644 index 00000000..d6a04daa --- /dev/null +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE2GrpcTestStrategy.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY + * All Rights Reserved. + */ + +package org.testcontainers.containers.integration.tqe; + +import java.io.IOException; +import java.util.List; +import java.util.Set; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.protobuf.ByteString; +import io.grpc.ManagedChannel; +import io.grpc.stub.StreamObserver; +import org.testcontainers.containers.utils.pojo.User; +import tarantool.queue_ee.v2.Consumer.SubscriptionNotifications; +import tarantool.queue_ee.v2.Consumer.SubscriptionRequest; +import tarantool.queue_ee.v2.ConsumerServiceGrpc; +import tarantool.queue_ee.v2.ConsumerServiceGrpc.ConsumerServiceStub; +import tarantool.queue_ee.v2.Publisher.BatchRequestMessage; +import tarantool.queue_ee.v2.Publisher.PublishBatchRequest; +import tarantool.queue_ee.v2.PublisherServiceGrpc; +import tarantool.queue_ee.v2.PublisherServiceGrpc.PublisherServiceBlockingStub; + +/** TQE 2.x gRPC API: {@code publishBatch} + unidirectional server-streaming subscribe. */ +final class TQE2GrpcTestStrategy implements GrpcTestStrategy { + + static final TQE2GrpcTestStrategy INSTANCE = new TQE2GrpcTestStrategy(); + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Override + public void publish(ManagedChannel channel, List users, String queue) throws Exception { + PublisherServiceBlockingStub pService = PublisherServiceGrpc.newBlockingStub(channel); + PublishBatchRequest.Builder requestBuilder = PublishBatchRequest.newBuilder(); + for (User user : users) { + requestBuilder.addMessages( + BatchRequestMessage.newBuilder() + .setPayload(ByteString.copyFrom(MAPPER.writeValueAsBytes(user)))); + } + pService.publishBatch(requestBuilder.setQueue(queue).build()); + } + + @Override + public void subscribe(ManagedChannel channel, String queue, Set result) { + ConsumerServiceStub cService = ConsumerServiceGrpc.newStub(channel); + cService.subscribe( + SubscriptionRequest.newBuilder().setCursor("").setQueue(queue).build(), + new StreamObserver<>() { + @Override + public void onNext(SubscriptionNotifications value) { + value.getNotificationsList().stream() + .map( + n -> { + try { + return MAPPER.readValue( + n.getMessage().getPayload().toByteArray(), User.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .forEach(result::add); + } + + @Override + public void onError(Throwable t) { + throw new RuntimeException("Stream observer received error", t); + } + + @Override + public void onCompleted() {} + }); + } +} diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java deleted file mode 100644 index 70c4f883..00000000 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3ClusterImplTest.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY - * All Rights Reserved. - */ - -package org.testcontainers.containers.integration.tqe; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.file.Path; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArraySet; -import java.util.concurrent.TimeUnit; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.protobuf.ByteString; -import io.grpc.ManagedChannel; -import io.grpc.stub.StreamObserver; -import org.instancio.Instancio; -import org.instancio.Select; -import org.instancio.generators.Generators; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.RepeatedTest; -import org.rnorth.ducttape.unreliables.Unreliables; -import org.testcontainers.containers.tqe.GrpcContainer; -import org.testcontainers.containers.tqe.GrpcContainer.GrpcRole; -import org.testcontainers.containers.tqe.TQE3ClusterImpl; -import org.testcontainers.containers.tqe.TQECluster; -import org.testcontainers.containers.tqe.configuration.FileTQEConfigurator; -import org.testcontainers.containers.tqe.configuration.TQEConfigurator; -import org.testcontainers.containers.utils.pojo.User; -import org.testcontainers.utility.DockerImageName; -import tarantool.queue_ee.Consumer.SubscriptionRequest; -import tarantool.queue_ee.Consumer.SubscriptionStreamRequest; -import tarantool.queue_ee.Consumer.SubscriptionStreamResponse; -import tarantool.queue_ee.ConsumerServiceGrpc; -import tarantool.queue_ee.ConsumerServiceGrpc.ConsumerServiceStub; -import tarantool.queue_ee.ProducerGrpc; -import tarantool.queue_ee.ProducerGrpc.ProducerBlockingStub; -import tarantool.queue_ee.ProducerOuterClass.ProduceMessage; -import tarantool.queue_ee.ProducerOuterClass.ProduceRequest; - -class TQE3ClusterImplTest { - - private static final DockerImageName IMAGE_NAME = - DockerImageName.parse( - System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") - + "tarantool/message-queue-ee:v3.5.0"); - - private static final Path SIMPLE_GRPC_CONFIG = - TQETestHelper.loadConfig("tqe3/simple-config/simple-grpc.yml"); - private static final Path SIMPLE_QUEUE_CONFIG = - TQETestHelper.loadConfig("tqe3/simple-config/simple-queue.yml"); - - @RepeatedTest(10) - void testPublishAndConsumeData() { - Assertions.assertDoesNotThrow( - () -> { - final ObjectMapper MAPPER = new ObjectMapper(); - - try (TQEConfigurator configurator = - FileTQEConfigurator.tqe3Builder( - IMAGE_NAME, SIMPLE_QUEUE_CONFIG, Set.of(SIMPLE_GRPC_CONFIG)) - .build(); - TQECluster cluster = new TQE3ClusterImpl(configurator)) { - cluster.start(); - - final String queueName = "test"; - - final List> producers = - cluster.grpc().values().stream() - .filter(g -> g.roles().contains(GrpcRole.PRODUCER)) - .toList(); - final List> consumers = - cluster.grpc().values().stream() - .filter(g -> g.roles().contains(GrpcRole.CONSUMER)) - .toList(); - - Assertions.assertFalse(producers.isEmpty()); - Assertions.assertFalse(consumers.isEmpty()); - - final Set grpcAddresses = producers.get(0).grpcAddresses(); - final Set consumerAddresses = consumers.get(0).grpcAddresses(); - - final Optional producerAddress = grpcAddresses.stream().findFirst(); - Assertions.assertTrue(producerAddress.isPresent()); - final Optional consumerAddress = - consumerAddresses.stream().findFirst(); - Assertions.assertTrue(consumerAddress.isPresent()); - - final ManagedChannel producerChannel = - TQETestHelper.createReadyChannel(producerAddress.get()); - final ManagedChannel consumerChannel = - TQETestHelper.createReadyChannel(consumerAddress.get()); - - try { - final ProducerBlockingStub producer = ProducerGrpc.newBlockingStub(producerChannel); - final ConsumerServiceStub consumer = ConsumerServiceGrpc.newStub(consumerChannel); - - final List users = - Instancio.ofList(User.class) - .size(100) - .generate( - Select.field(User::getName), - g -> g.string().alphaNumeric().allowEmpty().nullable()) - .generate(Select.field(User::getAge), Generators::ints) - .create(); - - Unreliables.retryUntilSuccess( - 60, - TimeUnit.SECONDS, - () -> { - final ProduceRequest.Builder requestBuilder = - ProduceRequest.newBuilder().setQueue(queueName); - for (User user : users) { - requestBuilder.addMessages( - ProduceMessage.newBuilder() - .setPayload(ByteString.copyFrom(MAPPER.writeValueAsBytes(user)))); - } - producer.produce(requestBuilder.build()); - return true; - }); - - final Set result = new CopyOnWriteArraySet<>(); - Unreliables.retryUntilSuccess( - 60, - TimeUnit.SECONDS, - () -> { - StreamObserver requestsStream = - consumer.subscribe( - new StreamObserver() { - @Override - public void onNext(SubscriptionStreamResponse response) { - response.getNotifications().getNotificationsList().stream() - .map( - n -> { - try { - return MAPPER.readValue( - n.getMessage().getPayload().toByteArray(), - User.class); - } catch (IOException e) { - throw new RuntimeException(e); - } - }) - .forEach(result::add); - } - - @Override - public void onError(Throwable t) { - throw new RuntimeException("Stream observer received error", t); - } - - @Override - public void onCompleted() {} - }); - requestsStream.onNext( - SubscriptionStreamRequest.newBuilder() - .setSubscribeRequest( - SubscriptionRequest.newBuilder().setCursor("").setQueue(queueName)) - .build()); - return true; - }); - - Unreliables.retryUntilTrue( - 60, TimeUnit.SECONDS, () -> new LinkedHashSet<>(users).size() == result.size()); - Assertions.assertEquals(new LinkedHashSet<>(users), result); - } finally { - consumerChannel.shutdownNow(); - producerChannel.shutdownNow(); - } - } - }); - } -} diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3GrpcTestStrategy.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3GrpcTestStrategy.java new file mode 100644 index 00000000..02e0a8c2 --- /dev/null +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQE3GrpcTestStrategy.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY + * All Rights Reserved. + */ + +package org.testcontainers.containers.integration.tqe; + +import java.io.IOException; +import java.util.List; +import java.util.Set; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.protobuf.ByteString; +import io.grpc.ManagedChannel; +import io.grpc.stub.StreamObserver; +import org.testcontainers.containers.utils.pojo.User; +import tarantool.queue_ee.Consumer.SubscriptionRequest; +import tarantool.queue_ee.Consumer.SubscriptionStreamRequest; +import tarantool.queue_ee.Consumer.SubscriptionStreamResponse; +import tarantool.queue_ee.ConsumerServiceGrpc; +import tarantool.queue_ee.ConsumerServiceGrpc.ConsumerServiceStub; +import tarantool.queue_ee.ProducerGrpc; +import tarantool.queue_ee.ProducerGrpc.ProducerBlockingStub; +import tarantool.queue_ee.ProducerOuterClass.ProduceMessage; +import tarantool.queue_ee.ProducerOuterClass.ProduceRequest; + +/** TQE 3.x gRPC API: {@code produce} + bidirectional client-streaming subscribe. */ +final class TQE3GrpcTestStrategy implements GrpcTestStrategy { + + static final TQE3GrpcTestStrategy INSTANCE = new TQE3GrpcTestStrategy(); + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Override + public void publish(ManagedChannel channel, List users, String queue) throws Exception { + ProducerBlockingStub producer = ProducerGrpc.newBlockingStub(channel); + ProduceRequest.Builder requestBuilder = ProduceRequest.newBuilder().setQueue(queue); + for (User user : users) { + requestBuilder.addMessages( + ProduceMessage.newBuilder() + .setPayload(ByteString.copyFrom(MAPPER.writeValueAsBytes(user)))); + } + producer.produce(requestBuilder.build()); + } + + @Override + public void subscribe(ManagedChannel channel, String queue, Set result) { + ConsumerServiceStub consumer = ConsumerServiceGrpc.newStub(channel); + StreamObserver requestsStream = + consumer.subscribe( + new StreamObserver<>() { + @Override + public void onNext(SubscriptionStreamResponse response) { + response.getNotifications().getNotificationsList().stream() + .map( + n -> { + try { + return MAPPER.readValue( + n.getMessage().getPayload().toByteArray(), User.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .forEach(result::add); + } + + @Override + public void onError(Throwable t) { + throw new RuntimeException("Stream observer received error", t); + } + + @Override + public void onCompleted() {} + }); + requestsStream.onNext( + SubscriptionStreamRequest.newBuilder() + .setSubscribeRequest(SubscriptionRequest.newBuilder().setCursor("").setQueue(queue)) + .build()); + } +} diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterFixture.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterFixture.java new file mode 100644 index 00000000..3334d3fd --- /dev/null +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterFixture.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY + * All Rights Reserved. + */ + +package org.testcontainers.containers.integration.tqe; + +import java.net.InetSocketAddress; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import io.grpc.ManagedChannel; +import org.testcontainers.containers.tqe.GrpcContainer; +import org.testcontainers.containers.tqe.GrpcContainer.GrpcRole; +import org.testcontainers.containers.tqe.TQECluster; +import org.testcontainers.containers.tqe.configuration.TQEConfigurator; + +/** + * Encapsulates the lifecycle of a {@link TQECluster} for a single test: builds the configurator, + * creates the cluster, starts it, and exposes helpers for resolving gRPC channels by role. {@link + * #close()} stops the cluster. + */ +final class TQEClusterFixture implements AutoCloseable { + + private final TQEVersion version; + private final TQEConfigurator configurator; + private final TQECluster cluster; + + TQEClusterFixture(TQEVersion version) { + this.version = version; + this.configurator = + version.configuratorBuilder(version.queueConfig(), Set.of(version.grpcConfig())).build(); + this.cluster = version.createCluster(configurator); + this.cluster.start(); + } + + TQEVersion version() { + return version; + } + + ManagedChannel createPublisherChannel() { + return createReadyChannel(findByRole(version.producerRole())); + } + + ManagedChannel createConsumerChannel() { + return createReadyChannel(findByRole(GrpcRole.CONSUMER)); + } + + void restart(long delayBefore, TimeUnit unitBefore, long delayAfter, TimeUnit unitAfter) + throws InterruptedException { + this.cluster.restart(delayBefore, unitBefore, delayAfter, unitAfter); + } + + @Override + public void close() { + this.cluster.stop(); + } + + private GrpcContainer findByRole(GrpcRole role) { + return this.cluster.grpc().values().stream() + .filter(g -> g.roles().contains(role)) + .findFirst() + .orElseThrow( + () -> + new IllegalStateException( + "No gRPC container with role " + + role + + " in cluster " + + cluster.clusterName())); + } + + private static ManagedChannel createReadyChannel(GrpcContainer grpc) { + InetSocketAddress address = + grpc.grpcAddresses().stream() + .findFirst() + .orElseThrow( + () -> new IllegalStateException("No gRPC address on container " + grpc.node())); + return TQETestHelper.createReadyChannel(address); + } +} diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterIntegrationTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterIntegrationTest.java new file mode 100644 index 00000000..f27e7c8c --- /dev/null +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterIntegrationTest.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY + * All Rights Reserved. + */ + +package org.testcontainers.containers.integration.tqe; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArraySet; +import java.util.concurrent.TimeUnit; + +import io.grpc.ManagedChannel; +import org.instancio.Instancio; +import org.instancio.Select; +import org.instancio.generators.Generators; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.rnorth.ducttape.unreliables.Unreliables; +import org.testcontainers.containers.utils.pojo.User; + +class TQEClusterIntegrationTest { + + private static final String QUEUE_NAME = "test"; + private static final int USERS_COUNT = 100; + private static final int RETRY_TIMEOUT_SECONDS = 60; + + @ParameterizedTest + @EnumSource(TQEVersion.class) + @DisplayName("publish → subscribe round-trip across TQE versions") + void testPublishAndConsumeData(TQEVersion version) { + Assertions.assertDoesNotThrow( + () -> { + try (TQEClusterFixture fx = new TQEClusterFixture(version)) { + ManagedChannel publisherChannel = fx.createPublisherChannel(); + ManagedChannel consumerChannel = fx.createConsumerChannel(); + try { + final List users = generateUsers(); + + Unreliables.retryUntilSuccess( + RETRY_TIMEOUT_SECONDS, + TimeUnit.SECONDS, + () -> { + version.strategy().publish(publisherChannel, users, QUEUE_NAME); + return true; + }); + + final Set result = new CopyOnWriteArraySet<>(); + Unreliables.retryUntilSuccess( + RETRY_TIMEOUT_SECONDS, + TimeUnit.SECONDS, + () -> { + version.strategy().subscribe(consumerChannel, QUEUE_NAME, result); + return true; + }); + + Unreliables.retryUntilTrue( + RETRY_TIMEOUT_SECONDS, + TimeUnit.SECONDS, + () -> new LinkedHashSet<>(users).size() == result.size()); + Assertions.assertEquals(new LinkedHashSet<>(users), result); + } finally { + consumerChannel.shutdownNow(); + publisherChannel.shutdownNow(); + } + } + }); + } + + private static List generateUsers() { + return Instancio.ofList(User.class) + .size(USERS_COUNT) + .generate( + Select.field(User::getName), g -> g.string().alphaNumeric().allowEmpty().nullable()) + .generate(Select.field(User::getAge), Generators::ints) + .create(); + } +} diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterTest.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterTest.java index bc7a6b55..1bea5e2b 100644 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterTest.java +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEClusterTest.java @@ -17,9 +17,9 @@ import java.util.stream.Stream; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import org.testcontainers.containers.ContainerLaunchException; import org.testcontainers.containers.tqe.TQECluster; @@ -28,30 +28,22 @@ class TQEClusterTest { @ParameterizedTest - @MethodSource("tqeVersions") - void testMultiplyRestart(TQETestHelper.TQETestParams p) throws Exception { - try (TQEConfigurator configurator = - p.configuratorBuilderFactory().apply(p.queueConfig(), Set.of(p.grpcConfig())).build(); - TQECluster cluster = p.clusterFactory().apply(configurator)) { - cluster.start(); + @EnumSource(TQEVersion.class) + void testStartupAndShutdown(TQEVersion version) { + try (TQEClusterFixture fx = new TQEClusterFixture(version)) { + // Fixture starts the cluster in its constructor; close() stops it. } } - @Test - void testRestartMethod() throws Exception { - for (TQETestHelper.TQETestParams p : TQETestHelper.tqeVersions().toList()) { - try (TQEConfigurator configurator = - p.configuratorBuilderFactory() - .apply(p.queueConfig(), Set.of(p.grpcConfig())) - .build(); - TQECluster cluster = p.clusterFactory().apply(configurator)) { - cluster.start(); - cluster.restart(1, TimeUnit.SECONDS, 1, TimeUnit.SECONDS); - } + @ParameterizedTest + @EnumSource(TQEVersion.class) + void testRestartMethod(TQEVersion version) throws Exception { + try (TQEClusterFixture fx = new TQEClusterFixture(version)) { + fx.restart(1, TimeUnit.SECONDS, 1, TimeUnit.SECONDS); } } - public static Stream dataForTestInvalidQueueConfigShouldThrow() { + public static Stream dataForTestInvalidQueueConfig() { final List invalidConfigs = Arrays.asList( // no required test-super user @@ -176,43 +168,44 @@ public static Stream dataForTestInvalidQueueConfigShouldThrow() { - uri: router:3301 """); - return invalidConfigs.stream() - .map( - s -> { - final Path testConfigPath = - TQETestHelper.TEST_TEMP_DIR.resolve(UUID.randomUUID().toString()); - try { - Files.writeString(testConfigPath, s); - return Arguments.of(testConfigPath); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + return TQEVersion.all() + .flatMap( + version -> + invalidConfigs.stream() + .map( + s -> { + final Path testConfigPath = + TQETestHelper.TEST_TEMP_DIR.resolve(UUID.randomUUID().toString()); + try { + Files.writeString(testConfigPath, s); + return Arguments.of(version, testConfigPath); + } catch (IOException e) { + throw new RuntimeException(e); + } + })); } @ParameterizedTest - @MethodSource("dataForTestInvalidQueueConfigShouldThrow") - void testInvalidQueueConfigShouldThrow(Path queueConfig) { - for (TQETestHelper.TQETestParams p : TQETestHelper.tqeVersions().toList()) { - Assertions.assertThrows( - ContainerLaunchException.class, - () -> { - try (TQEConfigurator configurator = - p.configuratorBuilderFactory() - .apply(queueConfig, Set.of(p.grpcConfig())) - .withStartupTimeout(Duration.ofSeconds(5)) - .build(); - TQECluster cluster = p.clusterFactory().apply(configurator)) { - cluster.start(); - } - }); - } + @MethodSource("dataForTestInvalidQueueConfig") + void testInvalidQueueConfig(TQEVersion version, Path queueConfig) { + Assertions.assertThrows( + ContainerLaunchException.class, + () -> { + try (TQEConfigurator configurator = + version + .configuratorBuilder(queueConfig, Set.of(version.grpcConfig())) + .withStartupTimeout(Duration.ofSeconds(5)) + .build(); + TQECluster cluster = version.createCluster(configurator)) { + cluster.start(); + } + }); } public static Stream dataForTestInvalidGrpcConfig() { - return TQETestHelper.tqeVersions() + return TQEVersion.all() .flatMap( - p -> { + version -> { final List invalidGrpcConfigs = Arrays.asList( // unknown host @@ -239,7 +232,7 @@ public static Stream dataForTestInvalidGrpcConfig() { storage: - "master:3301" """ - .formatted(p.producerRoleName()), + .formatted(version.producerRoleName()), // no consumers and producers """ core_port: 1111 @@ -264,7 +257,7 @@ public static Stream dataForTestInvalidGrpcConfig() { storage: - "master:3301" """ - .formatted(p.producerRoleName()), + .formatted(version.producerRoleName()), // no core_port parameter """ grpc_listen: @@ -288,7 +281,7 @@ public static Stream dataForTestInvalidGrpcConfig() { storage: - "master:3301" """ - .formatted(p.producerRoleName()), + .formatted(version.producerRoleName()), // no listen.uri parameter """ core_port: 1111 @@ -311,7 +304,7 @@ public static Stream dataForTestInvalidGrpcConfig() { storage: - "master:3301" """ - .formatted(p.producerRoleName())); + .formatted(version.producerRoleName())); return invalidGrpcConfigs.stream() .map( @@ -320,7 +313,7 @@ public static Stream dataForTestInvalidGrpcConfig() { TQETestHelper.TEST_TEMP_DIR.resolve(UUID.randomUUID() + ".yml"); try { Files.writeString(testConfigPath, s); - return Arguments.of(p, testConfigPath); + return Arguments.of(version, testConfigPath); } catch (IOException e) { throw new RuntimeException(e); } @@ -330,22 +323,18 @@ public static Stream dataForTestInvalidGrpcConfig() { @ParameterizedTest @MethodSource("dataForTestInvalidGrpcConfig") - void testInvalidGrpcConfig(TQETestHelper.TQETestParams p, Path grpcConfig) { + void testInvalidGrpcConfig(TQEVersion version, Path grpcConfig) { Assertions.assertThrows( ContainerLaunchException.class, () -> { try (TQEConfigurator configurator = - p.configuratorBuilderFactory() - .apply(p.queueConfig(), Set.of(grpcConfig)) + version + .configuratorBuilder(version.queueConfig(), Set.of(grpcConfig)) .withStartupTimeout(Duration.ofSeconds(5)) .build(); - TQECluster cluster = p.clusterFactory().apply(configurator)) { + TQECluster cluster = version.createCluster(configurator)) { cluster.start(); } }); } - - static Stream tqeVersions() { - return TQETestHelper.tqeVersions(); - } } diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQETestHelper.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQETestHelper.java index e412505e..88b96e5e 100644 --- a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQETestHelper.java +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQETestHelper.java @@ -12,21 +12,11 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Objects; -import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.function.BiFunction; -import java.util.function.Function; -import java.util.stream.Stream; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import org.rnorth.ducttape.unreliables.Unreliables; -import org.testcontainers.containers.tqe.TQE2ClusterImpl; -import org.testcontainers.containers.tqe.TQE3ClusterImpl; -import org.testcontainers.containers.tqe.TQECluster; -import org.testcontainers.containers.tqe.configuration.FileTQEConfigurator; -import org.testcontainers.containers.tqe.configuration.TQEConfigurator; -import org.testcontainers.utility.DockerImageName; final class TQETestHelper { @@ -40,21 +30,6 @@ final class TQETestHelper { } } - private static final DockerImageName IMAGE_NAME_2 = - DockerImageName.parse( - System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") - + "tarantool/message-queue-ee:2.5.3"); - - private static final DockerImageName IMAGE_NAME_3 = - DockerImageName.parse( - System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") - + "tarantool/message-queue-ee:v3.5.0"); - - private static final Path GRPC_CONFIG_2 = loadConfig("tqe2/simple-config/simple-grpc.yml"); - private static final Path QUEUE_CONFIG_2 = loadConfig("tqe2/simple-config/simple-queue.yml"); - private static final Path GRPC_CONFIG_3 = loadConfig("tqe3/simple-config/simple-grpc.yml"); - private static final Path QUEUE_CONFIG_3 = loadConfig("tqe3/simple-config/simple-queue.yml"); - private TQETestHelper() {} static Path loadConfig(String resourcePath) { @@ -97,42 +72,4 @@ static ManagedChannel createReadyChannel(InetSocketAddress address) { return ch; }); } - - static Stream tqeVersions() { - return Stream.of( - new TQETestParams( - "TQE2", - IMAGE_NAME_2, - QUEUE_CONFIG_2, - GRPC_CONFIG_2, - "publisher", - (qc, gc) -> FileTQEConfigurator.tqe2Builder(IMAGE_NAME_2, qc, gc), - TQE2ClusterImpl::new, - false), - new TQETestParams( - "TQE3", - IMAGE_NAME_3, - QUEUE_CONFIG_3, - GRPC_CONFIG_3, - "producer", - (qc, gc) -> FileTQEConfigurator.tqe3Builder(IMAGE_NAME_3, qc, gc), - TQE3ClusterImpl::new, - true)); - } - - record TQETestParams( - String displayName, - DockerImageName imageName, - Path queueConfig, - Path grpcConfig, - String producerRoleName, - BiFunction, FileTQEConfigurator.Builder> configuratorBuilderFactory, - Function clusterFactory, - boolean requiresConfigure) { - - @Override - public String toString() { - return displayName; - } - } } diff --git a/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEVersion.java b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEVersion.java new file mode 100644 index 00000000..15c22df4 --- /dev/null +++ b/testcontainers/src/test/java/org/testcontainers/containers/integration/tqe/TQEVersion.java @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2025 VK DIGITAL TECHNOLOGIES LIMITED LIABILITY COMPANY + * All Rights Reserved. + */ + +package org.testcontainers.containers.integration.tqe; + +import java.nio.file.Path; +import java.util.Set; +import java.util.stream.Stream; + +import org.testcontainers.containers.tqe.GrpcContainer.GrpcRole; +import org.testcontainers.containers.tqe.TQE2ClusterImpl; +import org.testcontainers.containers.tqe.TQE3ClusterImpl; +import org.testcontainers.containers.tqe.TQECluster; +import org.testcontainers.containers.tqe.configuration.FileTQEConfigurator; +import org.testcontainers.containers.tqe.configuration.TQEConfigurator; +import org.testcontainers.utility.DockerImageName; + +/** + * Encapsulates all version-specific aspects of a TQE test: image, configs, gRPC role names, + * builder/cluster factories, and the gRPC strategy. + * + *

Adding a new TQE version (e.g. TQE 4.x) is an Open/Closed-friendly change: add a new constant, + * override the abstract methods — no test code needs to change. + */ +enum TQEVersion { + TQE2("TQE 2.x", "publisher", GrpcRole.PUBLISHER, false, TQE2GrpcTestStrategy.INSTANCE) { + @Override + public DockerImageName imageName() { + return IMAGE_TQE2; + } + + @Override + public Path queueConfig() { + return QUEUE_CONFIG_TQE2; + } + + @Override + public Path grpcConfig() { + return GRPC_CONFIG_TQE2; + } + + @Override + public FileTQEConfigurator.Builder configuratorBuilder(Path queue, Set grpc) { + return FileTQEConfigurator.tqe2Builder(imageName(), queue, grpc); + } + + @Override + public TQECluster createCluster(TQEConfigurator configurator) { + return new TQE2ClusterImpl(configurator); + } + }, + + TQE3("TQE 3.x", "producer", GrpcRole.PRODUCER, true, TQE3GrpcTestStrategy.INSTANCE) { + @Override + public DockerImageName imageName() { + return IMAGE_TQE3; + } + + @Override + public Path queueConfig() { + return QUEUE_CONFIG_TQE3; + } + + @Override + public Path grpcConfig() { + return GRPC_CONFIG_TQE3; + } + + @Override + public FileTQEConfigurator.Builder configuratorBuilder(Path queue, Set grpc) { + return FileTQEConfigurator.tqe3Builder(imageName(), queue, grpc); + } + + @Override + public TQECluster createCluster(TQEConfigurator configurator) { + return new TQE3ClusterImpl(configurator); + } + }; + + private static final DockerImageName IMAGE_TQE2 = + DockerImageName.parse( + System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") + + "tarantool/message-queue-ee:2.5.3"); + + private static final DockerImageName IMAGE_TQE3 = + DockerImageName.parse( + System.getenv().getOrDefault("TARANTOOL_REGISTRY", "") + + "tarantool/message-queue-ee:v3.5.0"); + + private static final Path QUEUE_CONFIG_TQE2 = + TQETestHelper.loadConfig("tqe2/simple-config/simple-queue.yml"); + private static final Path GRPC_CONFIG_TQE2 = + TQETestHelper.loadConfig("tqe2/simple-config/simple-grpc.yml"); + private static final Path QUEUE_CONFIG_TQE3 = + TQETestHelper.loadConfig("tqe3/simple-config/simple-queue.yml"); + private static final Path GRPC_CONFIG_TQE3 = + TQETestHelper.loadConfig("tqe3/simple-config/simple-grpc.yml"); + + private final String displayName; + private final String producerRoleName; + private final GrpcRole producerRole; + private final boolean requiresConfigure; + private final GrpcTestStrategy strategy; + + TQEVersion( + String displayName, + String producerRoleName, + GrpcRole producerRole, + boolean requiresConfigure, + GrpcTestStrategy strategy) { + this.displayName = displayName; + this.producerRoleName = producerRoleName; + this.producerRole = producerRole; + this.requiresConfigure = requiresConfigure; + this.strategy = strategy; + } + + public String displayName() { + return displayName; + } + + public String producerRoleName() { + return producerRoleName; + } + + public GrpcRole producerRole() { + return producerRole; + } + + /** + * Whether manual orchestration of {@code configurator.configure()} is required between starting + * the queue and the gRPC containers. TQE 2.x auto-configures inside {@code + * startTarantoolCluster()}; TQE 3.x defers it to {@code startGrpcEndpoints()}. Tests that drive + * the configurator directly (without a cluster) use this flag. + */ + public boolean requiresConfigure() { + return requiresConfigure; + } + + public GrpcTestStrategy strategy() { + return strategy; + } + + public abstract DockerImageName imageName(); + + public abstract Path queueConfig(); + + public abstract Path grpcConfig(); + + public abstract FileTQEConfigurator.Builder configuratorBuilder(Path queue, Set grpc); + + public abstract TQECluster createCluster(TQEConfigurator configurator); + + static Stream all() { + return Stream.of(values()); + } + + @Override + public String toString() { + return displayName; + } +}