From c328503a703bd3ceafc125dd5e94e83cd980d844 Mon Sep 17 00:00:00 2001 From: vruttantmankad Date: Thu, 18 Apr 2024 18:06:30 +0530 Subject: [PATCH 1/7] added support for acl auth --- src/ziggurat/config.clj | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/ziggurat/config.clj b/src/ziggurat/config.clj index 9db7afaf..2e17def7 100644 --- a/src/ziggurat/config.clj +++ b/src/ziggurat/config.clj @@ -6,7 +6,8 @@ [mount.core :refer [defstate]] [ziggurat.util.java-util :as util]) (:import (java.util Properties) - [org.apache.kafka.common.config SaslConfigs]) + [org.apache.kafka.common.config SaslConfigs] + [org.apache.kafka.clients CommonClientConfigs]) (:gen-class :methods [^{:static true} [get [String] Object] @@ -199,7 +200,8 @@ (def jaas-template {"PLAIN" "org.apache.kafka.common.security.plain.PlainLoginModule" - "SCRAM-SHA-512" "org.apache.kafka.common.security.scram.ScramLoginModule"}) + "SCRAM-SHA-512" "org.apache.kafka.common.security.scram.ScramLoginModule" + "SCRAM-SHA-256" "org.apache.kafka.common.security.scram.ScramLoginModule"}) (defn create-jaas-properties [user-name password mechanism] @@ -211,10 +213,13 @@ (if (some? jaas-config) (let [username (get jaas-config :username) password (get jaas-config :password) - mechanism (get jaas-config :mechanism)] + mechanism (get jaas-config :mechanism) + protocol (get jaas-config :protocol) + jaas_props (create-jaas-properties username password mechanism)] (doto properties - (.put SaslConfigs/SASL_JAAS_CONFIG - (create-jaas-properties username password mechanism)))) + (.put SaslConfigs/SASL_JAAS_CONFIG jaas_props) + (.put SaslConfigs/SASL_MECHANISM mechanism) + (.put CommonClientConfigs/SECURITY_PROTOCOL_CONFIG protocol))) properties)) (defn build-ssl-properties From 095e02bdfdf8169ba19675ce05cc4bbf1201bd9c Mon Sep 17 00:00:00 2001 From: vruttantmankad Date: Fri, 19 Apr 2024 16:45:36 +0530 Subject: [PATCH 2/7] fix tests --- src/ziggurat/config.clj | 3 ++- test/ziggurat/config_test.clj | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ziggurat/config.clj b/src/ziggurat/config.clj index 2e17def7..4cfedcc7 100644 --- a/src/ziggurat/config.clj +++ b/src/ziggurat/config.clj @@ -239,7 +239,8 @@ :ssl-keystore-password <> {:jaas {:username <> :password <> - :mechanism <>}}} + :mechanism <> + :protocol <>}}} " (let [ssl-configs-enabled (:enabled ssl-config-map) jaas-config (get ssl-config-map :jaas)] diff --git a/test/ziggurat/config_test.clj b/test/ziggurat/config_test.clj index 1337bdd4..ac13c51a 100644 --- a/test/ziggurat/config_test.clj +++ b/test/ziggurat/config_test.clj @@ -323,7 +323,8 @@ :ssl-keystore-password "some-password" :jaas {:username "myuser" :password "mypassword" - :mechanism "SCRAM-SHA-512"}})] + :mechanism "SCRAM-SHA-512" + :protocol "PLAINTEXT"}})] (let [streams-config-map {:auto-offset-reset :latest} props (build-streams-config-properties streams-config-map) auto-offset-reset (.getProperty props "auto.offset.reset") From 3edc85ba4e160a3fae3b6d2150f46f9dc0d9ca07 Mon Sep 17 00:00:00 2001 From: vruttantmankad Date: Tue, 23 Apr 2024 14:40:50 +0530 Subject: [PATCH 3/7] code refactor --- src/ziggurat/config.clj | 50 ++++++++++++++++++++++++++++++----- test/ziggurat/config_test.clj | 21 ++++++++++++--- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/src/ziggurat/config.clj b/src/ziggurat/config.clj index 4cfedcc7..2a03f8be 100644 --- a/src/ziggurat/config.clj +++ b/src/ziggurat/config.clj @@ -96,6 +96,9 @@ (defn ssl-config [] (get-in config [:ziggurat :ssl])) +(defn sasl-config [] + (get-in config [:ziggurat :sasl])) + (defn rabbitmq-config [] (get (ziggurat-config) :rabbit-mq)) @@ -214,12 +217,17 @@ (let [username (get jaas-config :username) password (get jaas-config :password) mechanism (get jaas-config :mechanism) - protocol (get jaas-config :protocol) jaas_props (create-jaas-properties username password mechanism)] (doto properties - (.put SaslConfigs/SASL_JAAS_CONFIG jaas_props) - (.put SaslConfigs/SASL_MECHANISM mechanism) - (.put CommonClientConfigs/SECURITY_PROTOCOL_CONFIG protocol))) + (.put SaslConfigs/SASL_JAAS_CONFIG jaas_props))) + properties)) + +(defn- add-sasl-properties + [properties mechanism protocol] + (if (and (some? mechanism) (some? protocol)) + (doto properties + (.put SaslConfigs/SASL_MECHANISM mechanism) + (.put CommonClientConfigs/SECURITY_PROTOCOL_CONFIG protocol)) properties)) (defn build-ssl-properties @@ -250,6 +258,35 @@ (reduce-kv set-property-fn pr ssl-config-map)) properties))) +(defn build-sasl-properties + [properties set-property-fn sasl-config-map] + "Builds SASL properties from sasl-config-map which is a map where keys are + Clojure keywords in kebab case. These keys are converted to Kafka properties by set-property-fn. + + SASL properties are only set if [:ziggurat :sasl :enabled] returns true. + + Creates JAAS template if values are provided in the map provided agains this key sequence + [:ziggurat :ssl :jaas]. + + Example of sasl-config-map + {:enabled true + :protocol <> + {:jaas + {:username <> + :password <> + :mechanism}}} + " + (let [sasl-configs-enabled (:enabled sasl-config-map) + jaas-config (get sasl-config-map :jaas) + mechanism (get jaas-config :mechanism) + protocol (get sasl-config-map :protocol)] + (if (true? sasl-configs-enabled) + (as-> properties pr + (add-jaas-properties pr jaas-config) + (add-sasl-properties pr mechanism protocol) + (reduce-kv set-property-fn pr sasl-config-map)) + properties))) + (defn build-properties "Builds Properties object from the provided config-map which is a map where keys are Clojure keywords in kebab case. These keys are converted to Kafka properties by set-property-fn. @@ -270,8 +307,9 @@ " [set-property-fn config-map] (as-> (Properties.) pr - (build-ssl-properties pr set-property-fn (ssl-config)) - (reduce-kv set-property-fn pr config-map))) + (build-ssl-properties pr set-property-fn (ssl-config)) + (build-sasl-properties pr set-property-fn (sasl-config)) + (reduce-kv set-property-fn pr config-map))) (def build-consumer-config-properties (partial build-properties (partial set-property consumer-config-mapping-table))) diff --git a/test/ziggurat/config_test.clj b/test/ziggurat/config_test.clj index ac13c51a..dfbc708a 100644 --- a/test/ziggurat/config_test.clj +++ b/test/ziggurat/config_test.clj @@ -21,6 +21,7 @@ statsd-config ziggurat-config ssl-config + sasl-config create-jaas-properties]] [ziggurat.fixtures :as f]) (:import (java.util ArrayList Properties))) @@ -323,8 +324,7 @@ :ssl-keystore-password "some-password" :jaas {:username "myuser" :password "mypassword" - :mechanism "SCRAM-SHA-512" - :protocol "PLAINTEXT"}})] + :mechanism "SCRAM-SHA-512"}})] (let [streams-config-map {:auto-offset-reset :latest} props (build-streams-config-properties streams-config-map) auto-offset-reset (.getProperty props "auto.offset.reset") @@ -348,7 +348,22 @@ (is (= auto-offset-reset "latest")) (is (= ssl-ks-location "/some/location")) (is (= ssl-ks-password "some-password")) - (is (nil? sasl-jaas-config))))))) + (is (nil? sasl-jaas-config))))) + (testing "sasl properties create jaas template from the map provided in [:ziggurat :sasl :jaas]" + (with-redefs [sasl-config (constantly {:enabled true + :protocol "SASL_PLAINTEXT" + :jaas {:username "myuser" + :password "mypassword" + :mechanism "SCRAM-SHA-256"}})] + (let [streams-config-map {:auto-offset-reset :latest} + props (build-streams-config-properties streams-config-map) + auto-offset-reset (.getProperty props "auto.offset.reset") + sasl-jaas-config (.getProperty props "sasl.jaas.config") + sasl-protocol (.getProperty props "security.protocol") + sasl-mechanism (.getProperty props "sasl.mechanism")] + (is (= auto-offset-reset "latest")) + (is (= sasl-protocol "SASL_PLAINTEXT")) + (is (= sasl-jaas-config (create-jaas-properties "myuser" "mypassword" "SCRAM-SHA-256")))))))) (deftest test-set-property (testing "set-property with empty (with spaces) value" From 6abbb21e086a79827618272972db0d95969fe558 Mon Sep 17 00:00:00 2001 From: vruttantmankad Date: Mon, 29 Apr 2024 15:37:30 +0530 Subject: [PATCH 4/7] login module as config --- src/ziggurat/config.clj | 23 +++++++++-------------- test/ziggurat/config_test.clj | 10 ++++++---- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/ziggurat/config.clj b/src/ziggurat/config.clj index 2a03f8be..8ca3484f 100644 --- a/src/ziggurat/config.clj +++ b/src/ziggurat/config.clj @@ -201,23 +201,17 @@ (.setProperty p sk nv)))) p) -(def jaas-template - {"PLAIN" "org.apache.kafka.common.security.plain.PlainLoginModule" - "SCRAM-SHA-512" "org.apache.kafka.common.security.scram.ScramLoginModule" - "SCRAM-SHA-256" "org.apache.kafka.common.security.scram.ScramLoginModule"}) - (defn create-jaas-properties - [user-name password mechanism] - (let [jaas-template (get jaas-template mechanism)] - (format "%s required username=\"%s\" password=\"%s\";" jaas-template user-name password))) + [user-name password login-module] + (format "%s required username=\"%s\" password=\"%s\";" login-module user-name password)) (defn- add-jaas-properties [properties jaas-config] (if (some? jaas-config) (let [username (get jaas-config :username) password (get jaas-config :password) - mechanism (get jaas-config :mechanism) - jaas_props (create-jaas-properties username password mechanism)] + login-module (get jaas-config :login-module) + jaas_props (create-jaas-properties username password login-module)] (doto properties (.put SaslConfigs/SASL_JAAS_CONFIG jaas_props))) properties)) @@ -265,20 +259,21 @@ SASL properties are only set if [:ziggurat :sasl :enabled] returns true. - Creates JAAS template if values are provided in the map provided agains this key sequence - [:ziggurat :ssl :jaas]. + Creates JAAS template if values are provided in the map provided against this key sequence + [:ziggurat :sasl :jaas]. Example of sasl-config-map {:enabled true :protocol <> + :mechanism <> {:jaas {:username <> :password <> - :mechanism}}} + :login-module <>}}} " (let [sasl-configs-enabled (:enabled sasl-config-map) jaas-config (get sasl-config-map :jaas) - mechanism (get jaas-config :mechanism) + mechanism (get sasl-config-map :mechanism) protocol (get sasl-config-map :protocol)] (if (true? sasl-configs-enabled) (as-> properties pr diff --git a/test/ziggurat/config_test.clj b/test/ziggurat/config_test.clj index dfbc708a..48b76b93 100644 --- a/test/ziggurat/config_test.clj +++ b/test/ziggurat/config_test.clj @@ -322,9 +322,10 @@ (with-redefs [ssl-config (constantly {:enabled true :ssl-keystore-location "/some/location" :ssl-keystore-password "some-password" + :mechanism "SCRAM-SHA-512" :jaas {:username "myuser" :password "mypassword" - :mechanism "SCRAM-SHA-512"}})] + :login-module "org.apache.kafka.common.security.scram.ScramLoginModule"}})] (let [streams-config-map {:auto-offset-reset :latest} props (build-streams-config-properties streams-config-map) auto-offset-reset (.getProperty props "auto.offset.reset") @@ -334,7 +335,7 @@ (is (= auto-offset-reset "latest")) (is (= ssl-ks-location "/some/location")) (is (= ssl-ks-password "some-password")) - (is (= sasl-jaas-config (create-jaas-properties "myuser" "mypassword" "SCRAM-SHA-512")))))) + (is (= sasl-jaas-config (create-jaas-properties "myuser" "mypassword" "org.apache.kafka.common.security.scram.ScramLoginModule")))))) (testing "ssl properties DO NOT create jaas template if no value is provided for key sequence [:ziggurat :ssl :jaas]" (with-redefs [ssl-config (constantly {:enabled true :ssl-keystore-location "/some/location" @@ -352,9 +353,10 @@ (testing "sasl properties create jaas template from the map provided in [:ziggurat :sasl :jaas]" (with-redefs [sasl-config (constantly {:enabled true :protocol "SASL_PLAINTEXT" + :mechanism "SCRAM-SHA-256" :jaas {:username "myuser" :password "mypassword" - :mechanism "SCRAM-SHA-256"}})] + :login-module "org.apache.kafka.common.security.scram.ScramLoginModule"}})] (let [streams-config-map {:auto-offset-reset :latest} props (build-streams-config-properties streams-config-map) auto-offset-reset (.getProperty props "auto.offset.reset") @@ -363,7 +365,7 @@ sasl-mechanism (.getProperty props "sasl.mechanism")] (is (= auto-offset-reset "latest")) (is (= sasl-protocol "SASL_PLAINTEXT")) - (is (= sasl-jaas-config (create-jaas-properties "myuser" "mypassword" "SCRAM-SHA-256")))))))) + (is (= sasl-jaas-config (create-jaas-properties "myuser" "mypassword" "org.apache.kafka.common.security.scram.ScramLoginModule")))))))) (deftest test-set-property (testing "set-property with empty (with spaces) value" From f655668d96df23d0ab5613226306ed3102f364d1 Mon Sep 17 00:00:00 2001 From: vruttantmankad Date: Mon, 29 Apr 2024 15:42:53 +0530 Subject: [PATCH 5/7] lint fix --- test/ziggurat/config_test.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/ziggurat/config_test.clj b/test/ziggurat/config_test.clj index 48b76b93..581056df 100644 --- a/test/ziggurat/config_test.clj +++ b/test/ziggurat/config_test.clj @@ -354,9 +354,9 @@ (with-redefs [sasl-config (constantly {:enabled true :protocol "SASL_PLAINTEXT" :mechanism "SCRAM-SHA-256" - :jaas {:username "myuser" - :password "mypassword" - :login-module "org.apache.kafka.common.security.scram.ScramLoginModule"}})] + :jaas {:username "myuser" + :password "mypassword" + :login-module "org.apache.kafka.common.security.scram.ScramLoginModule"}})] (let [streams-config-map {:auto-offset-reset :latest} props (build-streams-config-properties streams-config-map) auto-offset-reset (.getProperty props "auto.offset.reset") From f9cf9ceb413dd2d2872a11bdd63060a667685eee Mon Sep 17 00:00:00 2001 From: vruttantmankad Date: Mon, 29 Apr 2024 15:58:15 +0530 Subject: [PATCH 6/7] fix lint --- src/ziggurat/config.clj | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ziggurat/config.clj b/src/ziggurat/config.clj index 8ca3484f..838f1a74 100644 --- a/src/ziggurat/config.clj +++ b/src/ziggurat/config.clj @@ -277,9 +277,9 @@ protocol (get sasl-config-map :protocol)] (if (true? sasl-configs-enabled) (as-> properties pr - (add-jaas-properties pr jaas-config) - (add-sasl-properties pr mechanism protocol) - (reduce-kv set-property-fn pr sasl-config-map)) + (add-jaas-properties pr jaas-config) + (add-sasl-properties pr mechanism protocol) + (reduce-kv set-property-fn pr sasl-config-map)) properties))) (defn build-properties @@ -302,9 +302,9 @@ " [set-property-fn config-map] (as-> (Properties.) pr - (build-ssl-properties pr set-property-fn (ssl-config)) - (build-sasl-properties pr set-property-fn (sasl-config)) - (reduce-kv set-property-fn pr config-map))) + (build-ssl-properties pr set-property-fn (ssl-config)) + (build-sasl-properties pr set-property-fn (sasl-config)) + (reduce-kv set-property-fn pr config-map))) (def build-consumer-config-properties (partial build-properties (partial set-property consumer-config-mapping-table))) From 61c3418939b9eb829e16865ec233273e9aa17023 Mon Sep 17 00:00:00 2001 From: vruttantmankad Date: Mon, 29 Apr 2024 16:03:09 +0530 Subject: [PATCH 7/7] fix ssl config map --- src/ziggurat/config.clj | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/ziggurat/config.clj b/src/ziggurat/config.clj index 838f1a74..663ed020 100644 --- a/src/ziggurat/config.clj +++ b/src/ziggurat/config.clj @@ -239,16 +239,20 @@ {:enabled true :ssl-keystore-location <> :ssl-keystore-password <> + :mechanism <> + :protocol <> {:jaas {:username <> :password <> - :mechanism <> - :protocol <>}}} + :login-module <>}}} " (let [ssl-configs-enabled (:enabled ssl-config-map) - jaas-config (get ssl-config-map :jaas)] + jaas-config (get ssl-config-map :jaas) + mechanism (get ssl-config-map :mechanism) + protocol (get ssl-config-map :protocol)] (if (true? ssl-configs-enabled) (as-> properties pr (add-jaas-properties pr jaas-config) + (add-sasl-properties pr mechanism protocol) (reduce-kv set-property-fn pr ssl-config-map)) properties)))