Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add RabbitMQ Stream service connection from RabbitMQContainer #42443

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.springframework.amqp.rabbit.config.ContainerCustomizer;
import org.springframework.amqp.support.converter.MessageConverter;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.boot.autoconfigure.amqp.RabbitProperties.Stream;
import org.springframework.boot.autoconfigure.amqp.RabbitProperties.StreamContainer;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
Expand All @@ -50,6 +51,12 @@
@ConditionalOnClass(StreamRabbitListenerContainerFactory.class)
class RabbitStreamConfiguration {

@Bean
@ConditionalOnMissingBean(RabbitStreamConnectionDetails.class)
RabbitStreamConnectionDetails rabbitStreamConnectionDetails(RabbitProperties rabbitProperties) {
return new PropertiesRabbitStreamConnectionDetails(rabbitProperties.getStream());
}

@Bean(name = "rabbitListenerContainerFactory")
@ConditionalOnMissingBean(name = "rabbitListenerContainerFactory")
@ConditionalOnProperty(prefix = "spring.rabbitmq.listener", name = "type", havingValue = "stream")
Expand All @@ -68,9 +75,9 @@ StreamRabbitListenerContainerFactory streamRabbitListenerContainerFactory(Enviro

@Bean(name = "rabbitStreamEnvironment")
@ConditionalOnMissingBean(name = "rabbitStreamEnvironment")
Environment rabbitStreamEnvironment(RabbitProperties properties,
Environment rabbitStreamEnvironment(RabbitProperties properties, RabbitStreamConnectionDetails connectionDetails,
ObjectProvider<EnvironmentBuilderCustomizer> customizers) {
EnvironmentBuilder builder = configure(Environment.builder(), properties);
EnvironmentBuilder builder = configure(Environment.builder(), properties, connectionDetails);
customizers.orderedStream().forEach((customizer) -> customizer.customize(builder));
return builder.build();
}
Expand Down Expand Up @@ -99,12 +106,13 @@ RabbitStreamTemplate rabbitStreamTemplate(Environment rabbitStreamEnvironment, R
return template;
}

static EnvironmentBuilder configure(EnvironmentBuilder builder, RabbitProperties properties) {
static EnvironmentBuilder configure(EnvironmentBuilder builder, RabbitProperties properties,
RabbitStreamConnectionDetails connectionDetails) {
builder.lazyInitialization(true);
RabbitProperties.Stream stream = properties.getStream();
PropertyMapper map = PropertyMapper.get();
map.from(stream.getHost()).to(builder::host);
map.from(stream.getPort()).to(builder::port);
map.from(connectionDetails.getHost()).to(builder::host);
map.from(connectionDetails.getPort()).to(builder::port);
map.from(stream.getVirtualHost())
.as(withFallback(properties::getVirtualHost))
.whenNonNull()
Expand All @@ -118,4 +126,24 @@ private static Function<String, String> withFallback(Supplier<String> fallback)
return (value) -> (value != null) ? value : fallback.get();
}

static class PropertiesRabbitStreamConnectionDetails implements RabbitStreamConnectionDetails {

private final Stream streamProperties;

PropertiesRabbitStreamConnectionDetails(Stream streamProperties) {
this.streamProperties = streamProperties;
}

@Override
public String getHost() {
return this.streamProperties.getHost();
}

@Override
public int getPort() {
return this.streamProperties.getPort();
}

}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Copyright 2012-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.springframework.boot.autoconfigure.amqp;

import org.springframework.boot.autoconfigure.service.connection.ConnectionDetails;

/**
* Details required to establish a connection to a RabbitMQ service.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need this? Especially given the javadoc, it looks like to me that the existing RabbitConnectionDetails should be enough, isn't it?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

rabbitmq stream uses a different port and in order to do that the plugin should be enabled. Having two implementations of RabbitConnectionDetails will lead to having an exception with 2 beans unless other beans or configuration are disabled when stream are enabled.

I think would be possible to have a different virtual host for stream in a single RabbitMQ instance, so, having a specific ConnectionDetails would make sense.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think there's a bug in the current code as we're not taking RabbitConnectionDetails into account when falling back from stream-specific configuration to general Rabbit configuration. I've opened #42489 for that. When fixed, we'll fallback from spring.rabbit.stream.* properties to RabbitConnectionDetails. With the changes here in place, we'd then fallback from RabbitStreamConnectionDetails (a replacement for some of the spring.rabbit.stream.* properties) to RabbitConnectionDetails (a replacement for some of the general spring.rabbit.* properties).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thinking about this some more, Rabbit Streams support is deceptively complicated (to me at least) as I think it's the first time where different users may want different things. One user may want RabbitConnectionDetails and RabbitStreamConnectionDetails to be sourced from the same container as their app is using the same Rabbit broker for both streaming and regular messaging. Another user may want RabbitConnectionDetails and RabbitStreamConnectionDetails to be sourced from different containers as their app is using different Rabbit brokers for streaming and regular messaging.

With only RabbitMQContainer to look at, even if we could detect in the container connection details factory that the streams plugin is enabled, we wouldn't know if the user wanted this container to be used only for streams or for both streams and regular messaging. It feels like we may need some other signal to indicate how the container should be used as a source for connection details. This reminds me a bit of the exploratory work being done for SSL and the possibility of having SSL-related annotations on the container field. I'll flag this one for a meeting so I can see what the rest of the team thinks.

Copy link
Member

@wilkinsona wilkinsona Oct 9, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We think we may be able to check to see if the default streams port is exposed by the container. If it isn't, the factory should return null.

We should also think about Docker Compose as, ideally, we'd keep the supported services in sync across Compose and Testcontainers.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Upon further thought, I'm not sure that this will work for all cases. Those cases are:

  1. One container to be used for both streams and "standard" messaging
  2. Two containers: one for streams and one for standard messaging

In the two container case, RabbitContainerConnectionDetailsFactory will match a container with the streams port exposed and I think we'll end up with a duplicate connection details failure. If we made RabbitContainerConnectionDetailsFactory ignore containers with the streams port exposed it would fix the two container case but it would break the one container case.

Some experimentation's needed to see if the above's accurate.

*
* @author Eddú Meléndez
* @since 3.4.0
*/
public interface RabbitStreamConnectionDetails extends ConnectionDetails {
mhalbritter marked this conversation as resolved.
Show resolved Hide resolved

/**
* Rabbit server host.
* @return the rabbit server host
*/
String getHost();

/**
* Rabbit server port.
* @return the rabbit server port
*/
int getPort();

}
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@
package org.springframework.boot.autoconfigure.amqp;

import java.time.Duration;
import java.util.List;

import com.rabbitmq.stream.Address;
import com.rabbitmq.stream.BackOffDelayPolicy;
import com.rabbitmq.stream.Codec;
import com.rabbitmq.stream.Environment;
Expand All @@ -32,6 +34,7 @@
import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
import org.springframework.amqp.support.converter.MessageConverter;
import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.autoconfigure.amqp.RabbitStreamConfiguration.PropertiesRabbitStreamConnectionDetails;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
Expand All @@ -43,6 +46,7 @@
import org.springframework.rabbit.stream.producer.ProducerCustomizer;
import org.springframework.rabbit.stream.producer.RabbitStreamTemplate;
import org.springframework.rabbit.stream.support.converter.StreamMessageConverter;
import org.springframework.test.util.ReflectionTestUtils;

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.BDDMockito.then;
Expand Down Expand Up @@ -127,7 +131,7 @@ void whenCustomMessageListenerContainerFactoryIsDefinedThenAutoConfiguredContain
void environmentUsesPropertyDefaultsByDefault() {
EnvironmentBuilder builder = mock(EnvironmentBuilder.class);
RabbitProperties properties = new RabbitProperties();
RabbitStreamConfiguration.configure(builder, properties);
RabbitStreamConfiguration.configure(builder, properties, getRabbitConnectionDetails(properties));
then(builder).should().port(5552);
then(builder).should().host("localhost");
then(builder).should().lazyInitialization(true);
Expand All @@ -141,7 +145,7 @@ void whenStreamPortIsSetThenEnvironmentUsesCustomPort() {
EnvironmentBuilder builder = mock(EnvironmentBuilder.class);
RabbitProperties properties = new RabbitProperties();
properties.getStream().setPort(5553);
RabbitStreamConfiguration.configure(builder, properties);
RabbitStreamConfiguration.configure(builder, properties, getRabbitConnectionDetails(properties));
then(builder).should().port(5553);
}

Expand All @@ -150,7 +154,7 @@ void whenStreamHostIsSetThenEnvironmentUsesCustomHost() {
EnvironmentBuilder builder = mock(EnvironmentBuilder.class);
RabbitProperties properties = new RabbitProperties();
properties.getStream().setHost("stream.rabbit.example.com");
RabbitStreamConfiguration.configure(builder, properties);
RabbitStreamConfiguration.configure(builder, properties, getRabbitConnectionDetails(properties));
then(builder).should().host("stream.rabbit.example.com");
}

Expand All @@ -159,7 +163,7 @@ void whenStreamVirtualHostIsSetThenEnvironmentUsesCustomVirtualHost() {
EnvironmentBuilder builder = mock(EnvironmentBuilder.class);
RabbitProperties properties = new RabbitProperties();
properties.getStream().setVirtualHost("stream-virtual-host");
RabbitStreamConfiguration.configure(builder, properties);
RabbitStreamConfiguration.configure(builder, properties, getRabbitConnectionDetails(properties));
then(builder).should().virtualHost("stream-virtual-host");
}

Expand All @@ -168,7 +172,7 @@ void whenStreamVirtualHostIsNotSetButDefaultVirtualHostIsSetThenEnvironmentUsesD
EnvironmentBuilder builder = mock(EnvironmentBuilder.class);
RabbitProperties properties = new RabbitProperties();
properties.setVirtualHost("default-virtual-host");
RabbitStreamConfiguration.configure(builder, properties);
RabbitStreamConfiguration.configure(builder, properties, getRabbitConnectionDetails(properties));
then(builder).should().virtualHost("default-virtual-host");
}

Expand All @@ -178,7 +182,7 @@ void whenStreamCredentialsAreNotSetThenEnvironmentUsesRabbitCredentials() {
RabbitProperties properties = new RabbitProperties();
properties.setUsername("alice");
properties.setPassword("secret");
RabbitStreamConfiguration.configure(builder, properties);
RabbitStreamConfiguration.configure(builder, properties, getRabbitConnectionDetails(properties));
then(builder).should().username("alice");
then(builder).should().password("secret");
}
Expand All @@ -191,7 +195,7 @@ void whenStreamCredentialsAreSetThenEnvironmentUsesStreamCredentials() {
properties.setPassword("secret");
properties.getStream().setUsername("bob");
properties.getStream().setPassword("confidential");
RabbitStreamConfiguration.configure(builder, properties);
RabbitStreamConfiguration.configure(builder, properties, getRabbitConnectionDetails(properties));
then(builder).should().username("bob");
then(builder).should().password("confidential");
}
Expand Down Expand Up @@ -260,6 +264,22 @@ void environmentCreatedByBuilderCanBeCustomized() {
});
}

@Test
@SuppressWarnings("unchecked")
void connectionDetailsAreApplied() {
this.contextRunner.withPropertyValues("spring.rabbitmq.stream.name:stream-test")
.withUserConfiguration(CustomConnectionDetails.class)
.run((context) -> assertThat(context.getBean(Environment.class))
.extracting((environment) -> (List<Address>) ReflectionTestUtils.getField(environment, "addresses"))
.extracting((address) -> address.get(0))
.extracting("host", "port")
.containsExactly("rabbitmq", 5555));
}

private RabbitStreamConnectionDetails getRabbitConnectionDetails(RabbitProperties properties) {
return new PropertiesRabbitStreamConnectionDetails(properties.getStream());
}

@Configuration(proxyBeanMethods = false)
static class TestConfiguration {

Expand Down Expand Up @@ -345,4 +365,24 @@ EnvironmentBuilderCustomizer customizerB() {

}

@Configuration(proxyBeanMethods = false)
static class CustomConnectionDetails {

@Bean
RabbitStreamConnectionDetails customRabbitMqStreamConnectionDetails() {
return new RabbitStreamConnectionDetails() {
@Override
public String getHost() {
return "rabbitmq";
}

@Override
public int getPort() {
return 5555;
}
};
}

}

}
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ dependencies {
dockerTestImplementation("org.springframework:spring-jms")
dockerTestImplementation("org.springframework:spring-r2dbc")
dockerTestImplementation("org.springframework.amqp:spring-rabbit")
dockerTestImplementation("org.springframework.amqp:spring-rabbit-stream")
dockerTestImplementation("org.springframework.data:spring-data-redis")
dockerTestImplementation("org.springframework.kafka:spring-kafka")
dockerTestImplementation("org.springframework.ldap:spring-ldap-core")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
/*
* Copyright 2012-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.springframework.boot.testcontainers.service.connection.amqp;

import java.time.Duration;
import java.util.ArrayList;
import java.util.List;

import com.rabbitmq.stream.Address;
import com.rabbitmq.stream.Environment;
import org.awaitility.Awaitility;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.RabbitMQContainer;
import org.testcontainers.images.builder.Transferable;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;

import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.EnvironmentBuilderCustomizer;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitStreamConnectionDetails;
import org.springframework.boot.testcontainers.service.connection.ServiceConnection;
import org.springframework.boot.testsupport.container.TestImage;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.rabbit.stream.producer.RabbitStreamTemplate;
import org.springframework.rabbit.stream.support.StreamAdmin;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;

import static org.assertj.core.api.Assertions.assertThat;

/**
* Tests for {@link RabbitStreamContainerConnectionDetailsFactory}.
*
* @author Eddú Meléndez
*/
@SpringJUnitConfig
@TestPropertySource(
properties = { "spring.rabbitmq.stream.name=stream.queue1", "spring.rabbitmq.listener.type=stream" })
@Testcontainers(disabledWithoutDocker = true)
class RabbitStreamContainerConnectionDetailsFactoryIntegrationTests {

private static final int RABBITMQ_STREAMS_PORT = 5552;

@Container
@ServiceConnection
static final RabbitMQContainer rabbit = getRabbitMqStreamContainer();

private static RabbitMQContainer getRabbitMqStreamContainer() {
RabbitMQContainer container = TestImage.container(RabbitMQContainer.class);
container.addExposedPorts(RABBITMQ_STREAMS_PORT);
var enabledPlugins = "[rabbitmq_stream,rabbitmq_prometheus].";
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We don't use var in the codebase.

container.withCopyToContainer(Transferable.of(enabledPlugins), "/etc/rabbitmq/enabled_plugins");
return container;
}

@Autowired(required = false)
private RabbitStreamConnectionDetails connectionDetails;

@Autowired
private RabbitStreamTemplate rabbitStreamTemplate;

@Autowired
private TestListener listener;

@Test
void connectionCanBeMadeToRabbitContainer() {
assertThat(this.connectionDetails).isNotNull();
this.rabbitStreamTemplate.convertAndSend("message");
Awaitility.waitAtMost(Duration.ofMinutes(4))
.untilAsserted(() -> assertThat(this.listener.messages).containsExactly("message"));

}

@Configuration(proxyBeanMethods = false)
@ImportAutoConfiguration(RabbitAutoConfiguration.class)
static class TestConfiguration {

@Bean
StreamAdmin streamAdmin(Environment env) {
return new StreamAdmin(env, sc -> {
sc.stream("stream.queue1").create();
});
}

@Bean
EnvironmentBuilderCustomizer environmentBuilderCustomizer() {
return env -> {
Address entrypoint = new Address(rabbit.getHost(), rabbit.getMappedPort(RABBITMQ_STREAMS_PORT));
env.addressResolver(address -> entrypoint);
};
}

@Bean
TestListener testListener() {
return new TestListener();
}

}

static class TestListener {

private final List<String> messages = new ArrayList<>();

@RabbitListener(queues = "stream.queue1")
void processMessage(String message) {
this.messages.add(message);
}

}

}
Loading
Loading