diff --git a/Dockerfile b/Dockerfile index cd90808..bdf7d43 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,6 @@ RUN apk --update upgrade && \ apk add curl ca-certificates && \ update-ca-certificates && \ rm -rf /var/cache/apk/* -ADD config /config +RUN mkdir config && touch /config/mapping.json ADD rabbit-amazon-forwarder / CMD ["/rabbit-amazon-forwarder"] diff --git a/Makefile b/Makefile index be07276..b600fab 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ build: release: build docker build -t airhelp/rabbit-amazon-forwarder -f Dockerfile . -push: release +push: test release docker push airhelp/rabbit-amazon-forwarder fmt: diff --git a/README.md b/README.md index d881f0e..8da8ea4 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ Application to forward messages from RabbitMQ to different Amazon services. ## Configuration +### Environment variables + Export environment variables: ```bash export MAPPING_FILE=/config/mapping.json @@ -16,30 +18,58 @@ export AWS_ACCESS_KEY_ID=access_key export AWS_SECRET_ACCESS_KEY=secret_key ``` -## Mapping +### Mapping file -Definition of forwarder->consumer pairs should be placed inside mapping file. Samples are located in `tests` directory. +Definition of forwarder->consumer pairs should be placed inside mapping file. +Sample of RabbitMQ -> SNS mapping file. All fields are required. +```json +[ + { + "source" : { + "type" : "RabbitMQ", + "name" : "test-rabbit", + "connection" : "amqp://guest:guest@localhost:5672/", + "topic" : "amq.topic", + "queue" : "test-queue", + "routing" : "#" + }, + "destination" : { + "type" : "SNS", + "name" : "test-sns", + "target" : "arn:aws:sns:eu-west-1:XXXXXXXX:test-forwarder" + } + } +] +``` +Samples are located in `examples` directory. -## Build +## Build docker image ```bash make release ``` -# Release +## Run +Using docker: ```bash -make push -docker tag airhelp/rabbit-amazon-forwarder airhelp/rabbit-amazon-forwarder:$VERSION -docker push airhelp/rabbit-amazon-forwarder:$VERSION + ``` -## Run +Using docker-compose: ```bash docker-compose up ``` +# Release + +```bash +make push +docker tag airhelp/rabbit-amazon-forwarder airhelp/rabbit-amazon-forwarder:$VERSION +docker push airhelp/rabbit-amazon-forwarder:$VERSION +``` + ## Supervisor Supervisor is a module which starts the consumer->forwarder pairs. diff --git a/common/common.go b/common/common.go deleted file mode 100644 index e937425..0000000 --- a/common/common.go +++ /dev/null @@ -1,14 +0,0 @@ -package common - -const ( - MappingFile = "MAPPING_FILE" -) - -type Item struct { - Type string `json:"type"` - Name string `json:"name"` - ConnectionURL string `json:"connection"` - ExchangeName string `json:"topic"` - QueueName string `json:"queue"` - RoutingKey string `json:"routing"` -} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..18cbb82 --- /dev/null +++ b/config/config.go @@ -0,0 +1,23 @@ +package config + +const ( + // MappingFile mapping file environment variable + MappingFile = "MAPPING_FILE" +) + +// RabbitEntry RabbitMQ mapping entry +type RabbitEntry struct { + Type string `json:"type"` + Name string `json:"name"` + ConnectionURL string `json:"connection"` + ExchangeName string `json:"topic"` + QueueName string `json:"queue"` + RoutingKey string `json:"routing"` +} + +// AmazonEntry SQS/SNS mapping entry +type AmazonEntry struct { + Type string `json:"type"` + Name string `json:"name"` + Target string `json:"target"` +} diff --git a/config/mapping.json b/config/mapping.json deleted file mode 100644 index 0d4f101..0000000 --- a/config/mapping.json +++ /dev/null @@ -1,2 +0,0 @@ -[ -] diff --git a/examples/rabbit_to_sns.json b/examples/rabbit_to_sns.json new file mode 100644 index 0000000..9a7f05b --- /dev/null +++ b/examples/rabbit_to_sns.json @@ -0,0 +1,17 @@ +[ + { + "source" : { + "type" : "RabbitMQ", + "name" : "test-rabbit", + "connection" : "amqp://hrhisghq:ywKQs7GFfM5axHlExVRvZet5LLatD4sJ@squirrel.rmq.cloudamqp.com/hrhisghq", + "topic" : "amq.topic", + "queue" : "test-queue", + "routing" : "#" + }, + "destination" : { + "type" : "SNS", + "name" : "test-sns", + "target" : "arn:aws:sns:eu-west-1:XXXXXXXX:test-forwarder" + } + } +] diff --git a/examples/rabbit_to_sqs.json b/examples/rabbit_to_sqs.json new file mode 100644 index 0000000..8021402 --- /dev/null +++ b/examples/rabbit_to_sqs.json @@ -0,0 +1,17 @@ +[ + { + "source" : { + "type" : "RabbitMQ", + "name" : "test-rabbit", + "connection" : "amqp://guest:guest@localhost:5672/", + "topic" : "amq.topic", + "queue" : "test-queue", + "routing" : "#" + }, + "destination" : { + "type" : "SQS", + "name" : "test-queue", + "target" : "https://sqs.eu-west-1.amazonaws.com/XXXXXXXXX/test-queue" + } + } +] diff --git a/img/rabbit-amazon-forwarder.png b/img/rabbit-amazon-forwarder.png index 76dd80b..473e607 100644 Binary files a/img/rabbit-amazon-forwarder.png and b/img/rabbit-amazon-forwarder.png differ diff --git a/mapping/mapping.go b/mapping/mapping.go index 2086a65..b90385e 100644 --- a/mapping/mapping.go +++ b/mapping/mapping.go @@ -6,7 +6,7 @@ import ( "log" "os" - "github.com/AirHelp/rabbit-amazon-forwarder/common" + "github.com/AirHelp/rabbit-amazon-forwarder/config" "github.com/AirHelp/rabbit-amazon-forwarder/consumer" "github.com/AirHelp/rabbit-amazon-forwarder/forwarder" "github.com/AirHelp/rabbit-amazon-forwarder/rabbitmq" @@ -17,8 +17,8 @@ import ( type pairs []pair type pair struct { - Source common.Item `json:"source"` - Destination common.Item `json:"destination"` + Source config.RabbitEntry `json:"source"` + Destination config.AmazonEntry `json:"destination"` } // Client mapping client @@ -28,8 +28,8 @@ type Client struct { // Helper interface for creating consumers and forwaders type Helper interface { - createConsumer(item common.Item) consumer.Client - createForwarder(item common.Item) forwarder.Client + createConsumer(entry config.RabbitEntry) consumer.Client + createForwarder(entry config.AmazonEntry) forwarder.Client } type helperImpl struct{} @@ -65,27 +65,27 @@ func (c Client) Load() (map[consumer.Client]forwarder.Client, error) { } func (c Client) loadFile() ([]byte, error) { - filePath := os.Getenv(common.MappingFile) + filePath := os.Getenv(config.MappingFile) log.Print("Loading mapping file: ", filePath) return ioutil.ReadFile(filePath) } -func (h helperImpl) createConsumer(item common.Item) consumer.Client { - log.Printf("Creating consumer: [%s, %s]", item.Type, item.Name) - switch item.Type { +func (h helperImpl) createConsumer(entry config.RabbitEntry) consumer.Client { + log.Printf("Creating consumer: [%s, %s]", entry.Type, entry.Name) + switch entry.Type { case rabbitmq.Type: - return rabbitmq.CreateConsumer(item) + return rabbitmq.CreateConsumer(entry) } return nil } -func (h helperImpl) createForwarder(item common.Item) forwarder.Client { - log.Printf("Creating forwarder: [%s, %s]", item.Type, item.Name) - switch item.Type { +func (h helperImpl) createForwarder(entry config.AmazonEntry) forwarder.Client { + log.Printf("Creating forwarder: [%s, %s]", entry.Type, entry.Name) + switch entry.Type { case sns.Type: - return sns.CreateForwarder(item) + return sns.CreateForwarder(entry) case sqs.Type: - return sqs.CreateForwarder(item) + return sqs.CreateForwarder(entry) } return nil } diff --git a/mapping/mapping_test.go b/mapping/mapping_test.go index 8ea16b7..ccbe852 100644 --- a/mapping/mapping_test.go +++ b/mapping/mapping_test.go @@ -5,7 +5,7 @@ import ( "os" "testing" - "github.com/AirHelp/rabbit-amazon-forwarder/common" + "github.com/AirHelp/rabbit-amazon-forwarder/config" "github.com/AirHelp/rabbit-amazon-forwarder/consumer" "github.com/AirHelp/rabbit-amazon-forwarder/forwarder" "github.com/AirHelp/rabbit-amazon-forwarder/rabbitmq" @@ -19,7 +19,7 @@ const ( ) func TestLoad(t *testing.T) { - os.Setenv(common.MappingFile, "../tests/rabbit_to_sns.json") + os.Setenv(config.MappingFile, "../tests/rabbit_to_sns.json") client := New(MockMappingHelper{}) var consumerForwarderMap map[consumer.Client]forwarder.Client var err error @@ -32,7 +32,7 @@ func TestLoad(t *testing.T) { } func TestLoadFile(t *testing.T) { - os.Setenv(common.MappingFile, "../tests/rabbit_to_sns.json") + os.Setenv(config.MappingFile, "../tests/rabbit_to_sns.json") client := New() data, err := client.loadFile() if err != nil { @@ -46,13 +46,13 @@ func TestLoadFile(t *testing.T) { func TestCreateConsumer(t *testing.T) { client := New() consumerName := "test-rabbit" - item := common.Item{Type: "RabbitMQ", + entry := config.RabbitEntry{Type: "RabbitMQ", Name: consumerName, ConnectionURL: "url", ExchangeName: "topic", QueueName: "test-queue", RoutingKey: "#"} - consumer := client.helper.createConsumer(item) + consumer := client.helper.createConsumer(entry) if consumer.Name() != consumerName { t.Errorf("wrong consumer name, expected %s, found %s", consumerName, consumer.Name()) } @@ -61,13 +61,11 @@ func TestCreateConsumer(t *testing.T) { func TestCreateForwarderSNS(t *testing.T) { client := New(MockMappingHelper{}) forwarderName := "test-sns" - item := common.Item{Type: "SNS", - Name: forwarderName, - ConnectionURL: "", - ExchangeName: "topic", - QueueName: "", - RoutingKey: "#"} - forwarder := client.helper.createForwarder(item) + entry := config.AmazonEntry{Type: "SNS", + Name: forwarderName, + Target: "arn", + } + forwarder := client.helper.createForwarder(entry) if forwarder.Name() != forwarderName { t.Errorf("wrong forwarder name, expected %s, found %s", forwarderName, forwarder.Name()) } @@ -76,13 +74,11 @@ func TestCreateForwarderSNS(t *testing.T) { func TestCreateForwarderSQS(t *testing.T) { client := New(MockMappingHelper{}) forwarderName := "test-sqs" - item := common.Item{Type: "SQS", - Name: forwarderName, - ConnectionURL: "", - ExchangeName: "", - QueueName: "test-queue", - RoutingKey: "#"} - forwarder := client.helper.createForwarder(item) + entry := config.AmazonEntry{Type: "SQS", + Name: forwarderName, + Target: "arn", + } + forwarder := client.helper.createForwarder(entry) if forwarder.Name() != forwarderName { t.Errorf("wrong forwarder name, expected %s, found %s", forwarderName, forwarder.Name()) } @@ -103,18 +99,18 @@ type MockSQSForwarder struct { type ErrorForwarder struct{} -func (h MockMappingHelper) createConsumer(item common.Item) consumer.Client { - if item.Type != rabbitmq.Type { +func (h MockMappingHelper) createConsumer(entry config.RabbitEntry) consumer.Client { + if entry.Type != rabbitmq.Type { return nil } return MockRabbitConsumer{} } -func (h MockMappingHelper) createForwarder(item common.Item) forwarder.Client { - switch item.Type { +func (h MockMappingHelper) createForwarder(entry config.AmazonEntry) forwarder.Client { + switch entry.Type { case sns.Type: - return MockSNSForwarder{item.Name} + return MockSNSForwarder{entry.Name} case sqs.Type: - return MockSQSForwarder{item.Name} + return MockSQSForwarder{entry.Name} } return ErrorForwarder{} } diff --git a/rabbitmq/consumer.go b/rabbitmq/consumer.go index 9f68aa4..012e101 100644 --- a/rabbitmq/consumer.go +++ b/rabbitmq/consumer.go @@ -4,7 +4,7 @@ import ( "fmt" "log" - "github.com/AirHelp/rabbit-amazon-forwarder/common" + "github.com/AirHelp/rabbit-amazon-forwarder/config" "github.com/AirHelp/rabbit-amazon-forwarder/consumer" "github.com/AirHelp/rabbit-amazon-forwarder/forwarder" "github.com/streadway/amqp" @@ -34,8 +34,8 @@ type workerParams struct { } // CreateConsumer creates conusmer from string map -func CreateConsumer(item common.Item) consumer.Client { - return Consumer{item.Name, item.ConnectionURL, item.ExchangeName, item.QueueName, item.RoutingKey} +func CreateConsumer(entry config.RabbitEntry) consumer.Client { + return Consumer{entry.Name, entry.ConnectionURL, entry.ExchangeName, entry.QueueName, entry.RoutingKey} } // Name consumer name diff --git a/sns/forwader.go b/sns/forwader.go index 305d7d0..349936a 100644 --- a/sns/forwader.go +++ b/sns/forwader.go @@ -3,7 +3,7 @@ package sns import ( "log" - "github.com/AirHelp/rabbit-amazon-forwarder/common" + "github.com/AirHelp/rabbit-amazon-forwarder/config" "github.com/AirHelp/rabbit-amazon-forwarder/forwarder" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" @@ -21,9 +21,9 @@ type Forwarder struct { } // CreateForwarder creates instance of forwarder -func CreateForwarder(item common.Item) forwarder.Client { +func CreateForwarder(entry config.AmazonEntry) forwarder.Client { client := awsClient() - forwarder := Forwarder{item.Name, client, item.ExchangeName} + forwarder := Forwarder{entry.Name, client, entry.Target} log.Print("Created forwarder: ", forwarder.Name()) return forwarder } diff --git a/sns/forwarder_test.go b/sns/forwarder_test.go index 44177ed..3504d0d 100644 --- a/sns/forwarder_test.go +++ b/sns/forwarder_test.go @@ -3,18 +3,16 @@ package sns import ( "testing" - "github.com/AirHelp/rabbit-amazon-forwarder/common" + "github.com/AirHelp/rabbit-amazon-forwarder/config" ) func TestCreateForwarder(t *testing.T) { - item := common.Item{Type: "SNS", - Name: "sns-test", - ConnectionURL: "", - ExchangeName: "topic", - QueueName: "", - RoutingKey: "#"} - forwarder := CreateForwarder(item) - if forwarder.Name() != item.Name { - t.Errorf("wrong forwarder name, expected:%s, found: %s", item.Name, forwarder.Name()) + entry := config.AmazonEntry{Type: "SNS", + Name: "sns-test", + Target: "arn", + } + forwarder := CreateForwarder(entry) + if forwarder.Name() != entry.Name { + t.Errorf("wrong forwarder name, expected:%s, found: %s", entry.Name, forwarder.Name()) } } diff --git a/sqs/forwader.go b/sqs/forwader.go index b01cea2..a1179b1 100644 --- a/sqs/forwader.go +++ b/sqs/forwader.go @@ -3,7 +3,7 @@ package sqs import ( "log" - "github.com/AirHelp/rabbit-amazon-forwarder/common" + "github.com/AirHelp/rabbit-amazon-forwarder/config" "github.com/AirHelp/rabbit-amazon-forwarder/forwarder" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" @@ -21,9 +21,9 @@ type Forwarder struct { } // CreateForwarder creates instance of forwarder -func CreateForwarder(item common.Item) forwarder.Client { +func CreateForwarder(entry config.AmazonEntry) forwarder.Client { client := awsClient() - forwarder := Forwarder{item.Name, client, item.QueueName} + forwarder := Forwarder{entry.Name, client, entry.Target} log.Print("Created forwarder: ", forwarder.Name()) return forwarder } diff --git a/sqs/forwarder_test.go b/sqs/forwarder_test.go index 907aa44..b564aec 100644 --- a/sqs/forwarder_test.go +++ b/sqs/forwarder_test.go @@ -3,18 +3,16 @@ package sqs import ( "testing" - "github.com/AirHelp/rabbit-amazon-forwarder/common" + "github.com/AirHelp/rabbit-amazon-forwarder/config" ) func TestCreateForwarder(t *testing.T) { - item := common.Item{Type: "SQS", - Name: "sqs-test", - ConnectionURL: "", - ExchangeName: "", - QueueName: "test-queue", - RoutingKey: "#"} - forwarder := CreateForwarder(item) - if forwarder.Name() != item.Name { - t.Errorf("wrong forwarder name, expected:%s, found: %s", item.Name, forwarder.Name()) + entry := config.AmazonEntry{Type: "SQS", + Name: "sqs-test", + Target: "arn", + } + forwarder := CreateForwarder(entry) + if forwarder.Name() != entry.Name { + t.Errorf("wrong forwarder name, expected:%s, found: %s", entry.Name, forwarder.Name()) } } diff --git a/tests/rabbit_to_sns.json b/tests/rabbit_to_sns.json index 769b5dd..dd3fb12 100644 --- a/tests/rabbit_to_sns.json +++ b/tests/rabbit_to_sns.json @@ -11,10 +11,7 @@ "destination" : { "type" : "SNS", "name" : "test-sns", - "connection" : "", - "topic" : "arn:aws:sns:eu-west-1:XXXXXXXX:test-forwarder", - "queue" : "", - "routing" : "" + "target" : "arn:aws:sns:eu-west-1:XXXXXXXX:test-forwarder" } } ] diff --git a/tests/rabbit_to_sqs.json b/tests/rabbit_to_sqs.json index 08357a5..8021402 100644 --- a/tests/rabbit_to_sqs.json +++ b/tests/rabbit_to_sqs.json @@ -11,10 +11,7 @@ "destination" : { "type" : "SQS", "name" : "test-queue", - "connection" : "", - "topic" : "", - "queue" : "https://sqs.eu-west-1.amazonaws.com/XXXXXXXXX/test-queue", - "routing" : "" + "target" : "https://sqs.eu-west-1.amazonaws.com/XXXXXXXXX/test-queue" } } ]