Merge tag 'v0.2.5'

taylorchu · Apr 19, 2022 · cec209f · cec209f
2 parents f84f9f2 + 92f53f2
commit cec209f
Show file tree

Hide file tree

Showing 26 changed files with 1,065 additions and 504 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,39 +1,49 @@
-version: 2
+version: 2.1
 jobs:
-  redis5:
+  redis:
+    parameters:
+      redis-image:
+        type: string
+      go-image:
+        type: string
     docker:
-      - image: circleci/golang:1.16
+      - image: << parameters.go-image >>
         environment:
           - GO111MODULE=on
-      - image: redis:5-alpine
+      - image: << parameters.redis-image >>
     working_directory: /go/src/github.com/taylorchu/work
     steps:
       - checkout
       - run: dockerize -wait tcp://:6379
       - run: go test -p 1 -v ./...
-  redis5-cluster:
+  redis-cluster:
+    parameters:
+      redis-image:
+        type: string
+      go-image:
+        type: string
     docker:
-      - image: circleci/golang:1.16
+      - image: << parameters.go-image >>
         environment:
           - GO111MODULE=on
           - REDIS_ADDR=:7000,:7001,:7002
-      - image: redis:5-alpine
+      - image: << parameters.redis-image >>
         command:
           - --cluster-enabled
           - "yes"
           - --port
           - "7000"
           - --cluster-config-file
           - 7000.conf
-      - image: redis:5-alpine
+      - image: << parameters.redis-image >>
         command:
           - --cluster-enabled
           - "yes"
           - --port
           - "7001"
           - --cluster-config-file
           - 7001.conf
-      - image: redis:5-alpine
+      - image: << parameters.redis-image >>
         command:
           - --cluster-enabled
           - "yes"
@@ -45,74 +55,42 @@ jobs:
     steps:
       - checkout
       - run: dockerize -wait tcp://:7000 -wait tcp://:7001 -wait tcp://:7002
-      - run: sudo apt-get install redis-tools
+      - run: sudo apt-get update && sudo apt-get install redis-tools
       - run: echo yes | redis-cli --cluster create 127.0.0.1:7000 127.0.0.1:7001 127.0.0.1:7002
       - run: go test -p 1 -v ./...
-  redis6:
+  fossa:
     docker:
-      - image: circleci/golang:1.16
-        environment:
-          - GO111MODULE=on
-      - image: redis:6-alpine
+      - image: alpine
     working_directory: /go/src/github.com/taylorchu/work
     steps:
       - checkout
-      - run: dockerize -wait tcp://:6379
-      - run: go test -p 1 -v ./...
-  redis6-cluster:
-    docker:
-      - image: circleci/golang:1.16
-        environment:
-          - GO111MODULE=on
-          - REDIS_ADDR=:7000,:7001,:7002
-      - image: redis:6-alpine
-        command:
-          - --cluster-enabled
-          - "yes"
-          - --port
-          - "7000"
-          - --cluster-config-file
-          - 7000.conf
-      - image: redis:6-alpine
-        command:
-          - --cluster-enabled
-          - "yes"
-          - --port
-          - "7001"
-          - --cluster-config-file
-          - 7001.conf
-      - image: redis:6-alpine
-        command:
-          - --cluster-enabled
-          - "yes"
-          - --port
-          - "7002"
-          - --cluster-config-file
-          - 7002.conf
-    working_directory: /go/src/github.com/taylorchu/work
-    steps:
-      - checkout
-      - run: dockerize -wait tcp://:7000 -wait tcp://:7001 -wait tcp://:7002
-      - run: sudo apt-get install redis-tools
-      - run: echo yes | redis-cli --cluster create 127.0.0.1:7000 127.0.0.1:7001 127.0.0.1:7002
-      - run: go test -p 1 -v ./...
-  keydb:
-    docker:
-      - image: circleci/golang:1.16
-        environment:
-          - GO111MODULE=on
-      - image: eqalpha/keydb
-    working_directory: /go/src/github.com/taylorchu/work
-    steps:
-      - checkout
-      - run: dockerize -wait tcp://:6379
-      - run: go test -p 1 -v ./...
+      - run: apk add --no-cache curl bash
+      - run: |
+          curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/master/install-latest.sh | bash
+      - run: fossa analyze
+      - run: fossa test
 workflows:
   version: 2
   build_and_test:
     jobs:
-      - redis5
-      - redis6
-      - redis5-cluster
-      - redis6-cluster
-      - keydb
+      - redis:
+          matrix:
+            parameters:
+              go-image:
+                - circleci/golang:1.17
+                - circleci/golang:1.16
+              redis-image:
+                - redis:5-alpine
+                - redis:6-alpine
+                - eqalpha/keydb
+      - redis-cluster:
+          matrix:
+            parameters:
+              go-image:
+                - circleci/golang:1.17
+                - circleci/golang:1.16
+              redis-image:
+                - redis:5-alpine
+                - redis:6-alpine
+                - eqalpha/keydb
+      - fossa
diff --git a/README.md b/README.md
@@ -2,43 +2,48 @@
 
 [![GoDoc](https://godoc.org/github.com/taylorchu/work?status.png)](https://godoc.org/github.com/taylorchu/work)
 [![Go Report Card](https://goreportcard.com/badge/github.com/taylorchu/work)](https://goreportcard.com/report/github.com/taylorchu/work)
-[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Ftaylorchu%2Fwork.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Ftaylorchu%2Fwork?ref=badge_shield)
+[![FOSSA Status](https://app.fossa.com/api/projects/custom%2B4257%2Fgit%40github.com%3Ataylorchu%2Fwork.git.svg?type=shield)](https://app.fossa.com/projects/custom%2B4257%2Fgit%40github.com%3Ataylorchu%2Fwork.git?ref=badge_shield)
 [![CircleCI](https://circleci.com/gh/taylorchu/work.svg?style=svg)](https://circleci.com/gh/taylorchu/work)
 
 Please see `cmd/` for enqueuer and worker demo.
 
 ## Improvements
 
 - [x] queue backend abstraction
-    - redis is still the default, but the new design allows custom queue implementation.
+  - redis is still the default, but the new design allows custom queue implementation.
 - [x] simplify the keyspace design of redis queue backend
-    - The new design uses 1 redis hash per job, and 1 redis sorted set for queue.
-    - [Interesting read](https://kirshatrov.com/2018/07/20/redis-job-queue/)
+  - The new design uses 1 redis hash per job, and 1 redis sorted set for queue.
+  - [Interesting read](https://kirshatrov.com/2018/07/20/redis-job-queue/)
 - [x] modular
-    - The core only catches panics, retries on failure, and waits if a queue is empty.
-    - All other [functionalities](https://kirshatrov.com/2019/01/03/state-of-background-jobs/)
-      are either removed or moved to separate middlewares.
+  - The core only catches panics, retries on failure, and waits if a queue is empty.
+  - All other [functionalities](https://kirshatrov.com/2019/01/03/state-of-background-jobs/)
+    are either removed or moved to separate middlewares.
 - [x] support binary payload/args with message pack.
 - [x] replace built-in UI with prometheus metrics (use grafana if you want dashboard).
 - [x] additional optimizations (alloc + bulk queue ops)
-    ```go
-    BenchmarkWorkerRunJob/work_v1_1-8         	    3000	    515957 ns/op
-    BenchmarkWorkerRunJob/work_v2_1-8         	    5000	    284516 ns/op
-    BenchmarkWorkerRunJob/work_v1_10-8        	    1000	   2136546 ns/op
-    BenchmarkWorkerRunJob/work_v2_10-8        	    5000	    367997 ns/op
-    BenchmarkWorkerRunJob/work_v1_100-8       	     100	  18234023 ns/op
-    BenchmarkWorkerRunJob/work_v2_100-8       	    1000	   1759186 ns/op
-    BenchmarkWorkerRunJob/work_v1_1000-8      	      10	 162110100 ns/op
-    BenchmarkWorkerRunJob/work_v2_1000-8      	     100	  12646080 ns/op
-    BenchmarkWorkerRunJob/work_v1_10000-8     	       1	1691287122 ns/op
-    BenchmarkWorkerRunJob/work_v2_10000-8     	      10	 144923087 ns/op
-    BenchmarkWorkerRunJob/work_v1_100000-8    	       1	17515722574 ns/op
-    BenchmarkWorkerRunJob/work_v2_100000-8    	       1	1502468637 ns/op
-    PASS
-    ok  	github.com/taylorchu/work	87.901s
-    ```
-- [ ] http enqueuer
-
+  ```go
+  BenchmarkWorkerRunJob/work_v1_1-8         	    3000	    515957 ns/op
+  BenchmarkWorkerRunJob/work_v2_1-8         	    5000	    284516 ns/op
+  BenchmarkWorkerRunJob/work_v1_10-8        	    1000	   2136546 ns/op
+  BenchmarkWorkerRunJob/work_v2_10-8        	    5000	    367997 ns/op
+  BenchmarkWorkerRunJob/work_v1_100-8       	     100	  18234023 ns/op
+  BenchmarkWorkerRunJob/work_v2_100-8       	    1000	   1759186 ns/op
+  BenchmarkWorkerRunJob/work_v1_1000-8      	      10	 162110100 ns/op
+  BenchmarkWorkerRunJob/work_v2_1000-8      	     100	  12646080 ns/op
+  BenchmarkWorkerRunJob/work_v1_10000-8     	       1	1691287122 ns/op
+  BenchmarkWorkerRunJob/work_v2_10000-8     	      10	 144923087 ns/op
+  BenchmarkWorkerRunJob/work_v1_100000-8    	       1	17515722574 ns/op
+  BenchmarkWorkerRunJob/work_v2_100000-8    	       1	1502468637 ns/op
+  PASS
+  ok  	github.com/taylorchu/work	87.901s
+  ```
+- [x] http server
+  - [x] delete job
+  - [x] create job
+  - [x] get job status
+  - [x] get queue metrics (kubernetes autoscaler integration with [keda metrics api scaler](https://keda.sh/docs/2.5/scalers/metrics-api/))
+  - [ ] OpenAPI spec
 
 ## License
-[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Ftaylorchu%2Fwork.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Ftaylorchu%2Fwork?ref=badge_large)
+
+[![FOSSA Status](https://app.fossa.com/api/projects/custom%2B4257%2Fgit%40github.com%3Ataylorchu%2Fwork.git.svg?type=large)](https://app.fossa.com/projects/custom%2B4257%2Fgit%40github.com%3Ataylorchu%2Fwork.git?ref=badge_large)
diff --git a/bench/go.mod b/bench/go.mod
@@ -1,12 +1,24 @@
 module github.com/taylorchu/work/bench
 
-go 1.16
+go 1.17
 
 require (
-	github.com/go-redis/redis/v8 v8.10.0
+	github.com/go-redis/redis/v8 v8.11.4
 	github.com/gocraft/work v0.5.2-0.20180912175354-c85b71e20062
 	github.com/gomodule/redigo v1.8.5
-	github.com/robfig/cron v1.2.0 // indirect
 	github.com/stretchr/testify v1.7.0
-	github.com/taylorchu/work v0.1.12-0.20210619192221-d3325e033bdf
+	github.com/taylorchu/work v0.2.4-0.20220210165902-787ae01fa4ea
+)
+
+require (
+	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/robfig/cron v1.2.0 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )