Merge pull request #100 from controlplaneio/clearer_points_output

Clearer points output
controlplaneio · May 12, 2020 · 7ec6eeb · 7ec6eeb
2 parents 57af337 + 223236f
commit 7ec6eeb
Show file tree

Hide file tree

Showing 7 changed files with 131 additions and 94 deletions.
diff --git a/README.md b/README.md
@@ -50,7 +50,7 @@ Kubesec is available as a:
 - [Kubernetes Admission Controller](https://github.com/controlplaneio/kubesec-webhook)
 - [Kubectl plugin](https://github.com/controlplaneio/kubectl-kubesec)
 
-Or install the latest commit from Github with `go get -u github.com/controlplaneio/kubesec/cmd/kubesec`
+Or install the latest commit from GitHub with `go get -u github.com/controlplaneio/kubesec/cmd/kubesec`
 
 #### Command line usage:
 
@@ -235,7 +235,7 @@ Thanks to our awesome contributors!
 
 ## Contributing
 
-Kubesecis Apache 2.0 licensed and accepts contributions via GitHub pull requests.
+Kubesec is Apache 2.0 licensed and accepts contributions via GitHub pull requests.
 
 When submitting bug reports please include as much details as possible:
 
@@ -255,6 +255,11 @@ Your feedback is always welcome!
 
 # Release Notes
 
+## 2.4.0
+
+- added passed to the JSON output
+- note: repo tests now require `jq` - **only concerns maintainers**
+
 ## 2.3.1
 
 - patch to accept form data from the <https://kubesec.io> webpage sample form

diff --git a/pkg/ruler/report.go b/pkg/ruler/report.go
@@ -10,6 +10,7 @@ type Report struct {
 
 type RuleScoring struct {
 	Critical []RuleRef `json:"critical,omitempty"`
+	Passed   []RuleRef `json:"passed,omitempty"`
 	Advise   []RuleRef `json:"advise,omitempty"`
 }
 
@@ -39,9 +40,8 @@ func (rr RuleRefCustomOrder) Less(i, j int) bool {
 		// no integer absolute fn in golang
 		if rr[i].Points > 0 || rr[j].Points > 0 {
 			return rr[i].Points > rr[j].Points
-		} else {
-			return rr[i].Points < rr[j].Points
 		}
+		return rr[i].Points < rr[j].Points
 	}
 	return rr[i].Selector < rr[j].Selector
 }
diff --git a/pkg/ruler/ruleset.go b/pkg/ruler/ruleset.go
@@ -5,17 +5,18 @@ import (
 	"crypto/sha256"
 	"encoding/json"
 	"fmt"
+	"os"
+	"runtime"
+	"sort"
+	"strings"
+	"sync"
+
 	"github.com/controlplaneio/kubesec/pkg/rules"
 	"github.com/ghodss/yaml"
 	"github.com/in-toto/in-toto-golang/in_toto"
 	"github.com/instrumenta/kubeval/kubeval"
 	"github.com/thedevsaddam/gojsonq"
 	"go.uber.org/zap"
-	"os"
-	"runtime"
-	"sort"
-	"strings"
-	"sync"
 )
 
 type Ruleset struct {
@@ -252,8 +253,8 @@ func NewRuleset(logger *zap.SugaredLogger) *Ruleset {
 func (rs *Ruleset) Run(fileBytes []byte) ([]Report, error) {
 	reports := make([]Report, 0)
 
-	isJson := json.Valid(fileBytes)
-	if isJson {
+	isJSON := json.Valid(fileBytes)
+	if isJSON {
 		report := rs.generateReport(fileBytes)
 		reports = append(reports, report)
 	} else {
@@ -312,6 +313,7 @@ func (rs *Ruleset) generateReport(json []byte) Report {
 		Score:  0,
 		Scoring: RuleScoring{
 			Advise:   make([]RuleRef, 0),
+			Passed:   make([]RuleRef, 0),
 			Critical: make([]RuleRef, 0),
 		},
 	}
@@ -373,6 +375,7 @@ func (rs *Ruleset) generateReport(json []byte) Report {
 			if ruleRef.Points >= 0 {
 				rs.logger.Debugf("positive score rule matched %v (%v points)", ruleRef.Selector, ruleRef.Points)
 				report.Score += ruleRef.Points
+				report.Scoring.Passed = append(report.Scoring.Passed, ruleRef)
 			}
 
 			if ruleRef.Points < 0 {
@@ -398,6 +401,7 @@ func (rs *Ruleset) generateReport(json []byte) Report {
 
 	// sort results into priority order
 	sort.Sort(RuleRefCustomOrder(report.Scoring.Critical))
+	sort.Sort(RuleRefCustomOrder(report.Scoring.Passed))
 	sort.Sort(RuleRefCustomOrder(report.Scoring.Advise))
 
 	return report

diff --git a/test/1_cli.bats b/test/1_cli.bats
@@ -11,162 +11,181 @@ teardown() {
 }
 
 @test "fails Pod with unconfined seccomp" {
-  run _app ${TEST_DIR}/asset/score-0-pod-seccomp-unconfined.yml
+  run _app "${TEST_DIR}/asset/score-0-pod-seccomp-unconfined.yml"
   assert_lt_zero_points
 }
 
 @test "fails with CAP_SYS_ADMIN" {
-  run _app ${TEST_DIR}/asset/score-0-cap-sys-admin.yml
+  run _app "${TEST_DIR}/asset/score-0-cap-sys-admin.yml"
   assert_lt_zero_points
 }
 
 @test "fails with CAP_CHOWN" {
-  run _app ${TEST_DIR}/asset/score-0-cap-chown.yml
+  run _app "${TEST_DIR}/asset/score-0-cap-chown.yml"
   assert_zero_points
 }
 
 @test "fails with CAP_SYS_ADMIN and CAP_CHOWN" {
-  run _app ${TEST_DIR}/asset/score-0-cap-sys-admin-and-cap-chown.yml
+  run _app "${TEST_DIR}/asset/score-0-cap-sys-admin-and-cap-chown.yml"
   assert_lt_zero_points
 }
 
 @test "passes with securityContext capabilities drop all" {
-  run _app ${TEST_DIR}/asset/score-1-cap-drop-all.yml
+  run _app "${TEST_DIR}/asset/score-1-cap-drop-all.yml"
   assert_gt_zero_points
 }
 
 @test "passes deployment with securitycontext readOnlyRootFilesystem" {
-  run _app ${TEST_DIR}/asset/score-1-dep-ro-root-fs.yml
+  run _app "${TEST_DIR}/asset/score-1-dep-ro-root-fs.yml"
   assert_gt_zero_points
 }
 
 @test "passes deployment with securitycontext runAsNonRoot" {
-  run _app ${TEST_DIR}/asset/score-1-dep-seccon-run-as-non-root.yml
+  run _app "${TEST_DIR}/asset/score-1-dep-seccon-run-as-non-root.yml"
   assert_gt_zero_points
 }
 
 @test "fails deployment with securitycontext runAsUser 1" {
-  run _app ${TEST_DIR}/asset/score-1-dep-seccon-run-as-user-1.yml
+  run _app "${TEST_DIR}/asset/score-1-dep-seccon-run-as-user-1.yml"
   assert_zero_points
 }
 
 @test "passes deployment with securitycontext runAsUser > 10000" {
-  run _app ${TEST_DIR}/asset/score-1-dep-seccon-run-as-user-10001.yml
+  run _app "${TEST_DIR}/asset/score-1-dep-seccon-run-as-user-10001.yml"
   assert_gt_zero_points
 }
 
 @test "fails deployment with empty security context" {
-  run _app ${TEST_DIR}/asset/score-1-dep-empty-security-context.yml
+  run _app "${TEST_DIR}/asset/score-1-dep-empty-security-context.yml"
   assert_zero_points
 }
 
 @test "fails deployment with invalid security context" {
-  run _app ${TEST_DIR}/asset/score-1-dep-invalid-security-context.yml
+  run _app "${TEST_DIR}/asset/score-1-dep-invalid-security-context.yml"
   assert_line --index 4 --regexp 'fake: Additional property fake is not allowed'
 }
 
 @test "passes deployment with cgroup resource limits" {
-  run _app ${TEST_DIR}/asset/score-1-dep-resource-limit-cpu.yml
+  run _app "${TEST_DIR}/asset/score-1-dep-resource-limit-cpu.yml"
   assert_gt_zero_points
 }
 
 @test "passes deployment with cgroup memory limits" {
-  run _app ${TEST_DIR}/asset/score-1-dep-resource-limit-memory.yml
+  run _app "${TEST_DIR}/asset/score-1-dep-resource-limit-memory.yml"
   assert_gt_zero_points
 }
 
 @test "passes StatefulSet with volumeClaimTemplate" {
-  run _app ${TEST_DIR}/asset/score-1-statefulset-volumeclaimtemplate.yml
+  run _app "${TEST_DIR}/asset/score-1-statefulset-volumeclaimtemplate.yml"
   assert_gt_zero_points
 }
 
 @test "fails StatefulSet with no security" {
-  run _app ${TEST_DIR}/asset/score-0-statefulset-no-sec.yml
+  run _app "${TEST_DIR}/asset/score-0-statefulset-no-sec.yml"
   assert_zero_points
 }
 
 @test "fails DaemonSet with securityContext.privileged = true" {
-  run _app ${TEST_DIR}/asset/score-0-daemonset-securitycontext-privileged.yml
+  run _app "${TEST_DIR}/asset/score-0-daemonset-securitycontext-privileged.yml"
   assert_lt_zero_points
 }
 
 @test "fails DaemonSet with mounted host docker.sock" {
-  run _app ${TEST_DIR}/asset/score-0-daemonset-mount-docker-socket.yml
+  run _app "${TEST_DIR}/asset/score-0-daemonset-mount-docker-socket.yml"
   assert_lt_zero_points
 }
 
 @test "passes Pod with apparmor annotation" {
-  run _app ${TEST_DIR}/asset/score-3-pod-apparmor.yaml
+  run _app "${TEST_DIR}/asset/score-3-pod-apparmor.yaml"
   assert_gt_zero_points
 }
 
 @test "fails Pod with unconfined seccomp for all containers" {
-  run _app ${TEST_DIR}/asset/score-0-pod-seccomp-unconfined.yml
+  run _app "${TEST_DIR}/asset/score-0-pod-seccomp-unconfined.yml"
   assert_lt_zero_points
 }
 
 @test "passes Pod with non-unconfined seccomp for all containers" {
-  run _app ${TEST_DIR}/asset/score-0-pod-seccomp-non-unconfined.yml
+  run _app "${TEST_DIR}/asset/score-0-pod-seccomp-non-unconfined.yml"
   assert_gt_zero_points
 }
 
 @test "fails DaemonSet with hostNetwork" {
-  run _app ${TEST_DIR}/asset/score-0-daemonset-host-network.yml
+  run _app "${TEST_DIR}/asset/score-0-daemonset-host-network.yml"
   assert_lt_zero_points
 }
 
 @test "fails DaemonSet with hostPid" {
-  run _app ${TEST_DIR}/asset/score-0-daemonset-host-pid.yml
+  run _app "${TEST_DIR}/asset/score-0-daemonset-host-pid.yml"
   assert_lt_zero_points
 }
 
 @test "fails DaemonSet with host docker.socket" {
-  run _app ${TEST_DIR}/asset/score-0-daemonset-volume-host-docker-socket.yml
+  run _app "${TEST_DIR}/asset/score-0-daemonset-volume-host-docker-socket.yml"
   assert_lt_zero_points
 }
 
 @test "passes Deployment with serviceaccountname" {
-  run _app ${TEST_DIR}/asset/score-2-dep-serviceaccount.yml
-
+  run _app "${TEST_DIR}/asset/score-2-dep-serviceaccount.yml"
   assert_gt_zero_points
 }
 
 @test "passes pod with serviceaccountname" {
-  run _app ${TEST_DIR}/asset/score-2-pod-serviceaccount.yml
-
+  run _app "${TEST_DIR}/asset/score-2-pod-serviceaccount.yml"
   assert_gt_zero_points
 }
 
 @test "fails deployment with allowPrivilegeEscalation" {
-  run _app ${TEST_DIR}/asset/allowPrivilegeEscalation.yaml
-
+  run _app "${TEST_DIR}/asset/allowPrivilegeEscalation.yaml"
   assert_lt_zero_points
 }
 
-@test "returns integer point score for specific response lines" {
-  # ordering of scoring rules output currently non-determinstic, to be ordered in #44
-  run \
-    _app ${TEST_DIR}/asset/score-2-pod-serviceaccount.yml
+@test "returns integer point score for each advice element" {
+  JSON=$(_app "${TEST_DIR}/asset/score-2-pod-serviceaccount.yml")
+
+  run jq -r .[].scoring.advise[].points <<<"${JSON}"
+
+  for SCORE in $output; do
+    assert bash -c "[[ $SCORE =~ ^[0-9]+$ ]]"
+  done
+}
+
+@test "returns an ordered point score for all advice" {
+  JSON=$(_app "${TEST_DIR}/asset/score-2-pod-serviceaccount.yml")
+
+  run jq -r .[].scoring.advise[].points <<<"${JSON}"
+
+  PREVIOUS=""
+  for CURRENT in $output; do
+    [ "${PREVIOUS}" = "" ] || assert [ "$CURRENT" -le "${PREVIOUS}" ]
+    PREVIOUS="${CURRENT}"
+  done
+}
+
+@test "returns integer point score for each pass element" {
+  JSON=$(_app "${TEST_DIR}/asset/score-5-pod-serviceaccount.yml")
+
+  run jq -r .[].scoring.passed[].points <<<"${JSON}"
 
-  for LINE in 11 16 21 26 31 36 41 46 51 56 61; do
-    assert_line --index ${LINE} --regexp '^.*"points": [0-9]+$'
+  for SCORE in $output; do
+    assert bash -c "[[ $SCORE =~ ^[0-9]+$ ]]"
   done
 }
 
-@test "returns an ordered point score for all responses" {
-  run \
-    _app ${TEST_DIR}/asset/score-2-pod-serviceaccount.yml
+@test "returns an ordered point score for all passed" {
+  JSON=$(_app "${TEST_DIR}/asset/score-5-pod-serviceaccount.yml")
 
-  assert_line --index 11 --regexp '^.*\"points\": 3$'
+  run jq -r .[].scoring.passed[].points <<<"${JSON}"
 
-  for LINE in 16 21 26 31 36 41 46 51 56 61; do
-    assert_line --index ${LINE} --regexp '^.*\"points\": 1$'
+  PREVIOUS=""
+  for CURRENT in $output; do
+    [ "${PREVIOUS}" = "" ] || assert [ "$CURRENT" -le "${PREVIOUS}" ]
+    PREVIOUS="${CURRENT}"
   done
 }
 
 @test "check critical and advisory points listed by magnitude" {
-  run \
-    _app ${TEST_DIR}/asset/critical-double.yml
+  run _app "${TEST_DIR}/asset/critical-double.yml"
 
   # criticals - magnitude sort/lowest number first
   assert_line --index 11 --regexp '^.*\"points\": -30$'
@@ -175,12 +194,11 @@ teardown() {
   # advisories - magnitude sort/highest number first
   assert_line --index 23 --regexp '^.*\"points\": 3$'
   assert_line --index 28 --regexp '^.*\"points\": 3$'
-  assert_line --index 33 --regexp '^.*\"points\": 1$'  
+  assert_line --index 33 --regexp '^.*\"points\": 1$'
 }
 
 @test "check critical and advisory points as multi-yaml" {
-  run \
-    _app ${TEST_DIR}/asset/critical-double-multiple.yml
+  run _app "${TEST_DIR}/asset/critical-double-multiple.yml"
 
   # report 1 - criticals - magnitude sort/lowest number first
   assert_line --index 11 --regexp '^.*\"points\": -30$'
@@ -189,7 +207,7 @@ teardown() {
   # report 1 - advisories - magnitude sort/highest number first
   assert_line --index 23 --regexp '^.*\"points\": 3$'
   assert_line --index 28 --regexp '^.*\"points\": 3$'
-  assert_line --index 33 --regexp '^.*\"points\": 1$'  
+  assert_line --index 33 --regexp '^.*\"points\": 1$'
 
   # report 2 - criticals - magnitude sort/lowest number first
   assert_line --index 93 --regexp '^.*\"points\": -30$'
@@ -198,30 +216,25 @@ teardown() {
   # report 2 - advisories - magnitude sort/highest number first
   assert_line --index 105 --regexp '^.*\"points\": 3$'
   assert_line --index 110 --regexp '^.*\"points\": 3$'
-  assert_line --index 115 --regexp '^.*\"points\": 1$'  
+  assert_line --index 115 --regexp '^.*\"points\": 1$'
 }
 
 @test "returns deterministic report output" {
-  run \
-    _app ${TEST_DIR}/asset/score-2-pod-serviceaccount.yml
-
+  run _app "${TEST_DIR}/asset/score-2-pod-serviceaccount.yml"
   assert_success
 
   RUN_1_SIGNATURE=$(echo "${output}" | sha1sum)
 
-  run \
-    _app ${TEST_DIR}/asset/score-2-pod-serviceaccount.yml
-
+  run _app "${TEST_DIR}/asset/score-2-pod-serviceaccount.yml"
   assert_success
 
   RUN_2_SIGNATURE=$(echo "${output}" | sha1sum)
 
-  run \
-    _app ${TEST_DIR}/asset/score-2-pod-serviceaccount.yml
-
+  run _app "${TEST_DIR}/asset/score-2-pod-serviceaccount.yml"
   assert_success
 
   RUN_3_SIGNATURE=$(echo "${output}" | sha1sum)
-  [ "${RUN_1_SIGNATURE}" == "${RUN_2_SIGNATURE}" ]
-  [ "${RUN_1_SIGNATURE}" == "${RUN_3_SIGNATURE}" ]
+
+  assert [ "${RUN_1_SIGNATURE}" = "${RUN_2_SIGNATURE}" ]
+  assert [ "${RUN_1_SIGNATURE}" = "${RUN_3_SIGNATURE}" ]
 }