From 30d755c4e2940aadf042169112ac7702945e8389 Mon Sep 17 00:00:00 2001 From: euanwm Date: Sat, 22 Feb 2025 20:06:31 +0000 Subject: [PATCH 1/7] simple query cache mutex stuff --- backend/dbtools/cache_handler.go | 79 ++++++++++++++++++++++---------- backend/dbtools/sortby.go | 26 ++++++++--- 2 files changed, 73 insertions(+), 32 deletions(-) diff --git a/backend/dbtools/cache_handler.go b/backend/dbtools/cache_handler.go index a4731ea2..65c96806 100644 --- a/backend/dbtools/cache_handler.go +++ b/backend/dbtools/cache_handler.go @@ -1,22 +1,51 @@ package dbtools import ( - "backend/enum" "backend/structs" + "sync" +) + +type QueryState int + +const ( + None QueryState = iota + Working + Partial + Completed ) type QueryCache struct { - Store []Query + HashStore sync.Map // [structs.LeaderboardPayload]Query } type Query struct { - Filter structs.LeaderboardPayload DataPositions []int + Status QueryState } // AddQuery - Adds a query to the cache. func (q *QueryCache) AddQuery(query structs.LeaderboardPayload, dataPositions []int) { - q.Store = append(q.Store, Query{Filter: query, DataPositions: dataPositions}) + q.HashStore.Store(query, Query{dataPositions, Completed}) +} + +func (q *QueryCache) InitQuery(query structs.LeaderboardPayload) { + q.HashStore.Store(query, Query{ + DataPositions: nil, + Status: Working, + }) +} + +func (q *QueryCache) QueryStatus(query structs.LeaderboardPayload) QueryState { + queryStuff, ok := q.HashStore.Load(query) + if !ok { + return None + } else { + query, ok := queryStuff.(Query) + if !ok { + panic("how the fuck did you fuck this up?") + } + return query.Status + } } // CheckQuery - Checks if the query has been run before, if so, return the data positions. @@ -24,31 +53,31 @@ func (q *QueryCache) AddQuery(query structs.LeaderboardPayload, dataPositions [] // A false value indicates that the query has not been run before. // And a false value with a non-nil slice indicates that a similar query has been run before and the data positions are returned that should be used to filer from. // Hoping that the last case makes things a bit faster to reduce having to run multiple startup caching queries. -func (q *QueryCache) CheckQuery(query structs.LeaderboardPayload) (bool, []int) { - for _, cacheQuery := range q.Store { - if cacheQuery.Filter == query { - return true, cacheQuery.DataPositions - } - // checks for exact match - if cacheQuery.Filter.SortBy == query.SortBy && cacheQuery.Filter.Federation == query.Federation && cacheQuery.Filter.WeightClass == query.WeightClass && cacheQuery.Filter.Year == query.Year { - return true, cacheQuery.DataPositions +func (q *QueryCache) CheckQuery(query structs.LeaderboardPayload) (QueryState, []int) { + loadedData, ok := q.HashStore.Load(query) + if ok { + storedQuery, ok := loadedData.(Query) + if ok { + return storedQuery.Status, storedQuery.DataPositions } } - // if we get here, we haven't found a match, so we'll do some partial matching - for _, cacheQuery := range q.Store { - // all years for the same total/sinclair, federation and weight class - if cacheQuery.Filter.SortBy == query.SortBy && cacheQuery.Filter.Federation == query.Federation && cacheQuery.Filter.WeightClass == query.WeightClass && cacheQuery.Filter.Year == enum.AllYearsStr { - return false, cacheQuery.DataPositions - } - // all years for the same total/sinclair, federation, and all gendered weight classes - if cacheQuery.Filter.SortBy == query.SortBy && cacheQuery.Filter.Federation == query.Federation && cacheQuery.Filter.Year == enum.AllYearsStr { - if query.WeightClass[0] == 'M' && cacheQuery.Filter.WeightClass == "MALL" { + /* + // if we get here, we haven't found a match, so we'll do some partial matching + for _, cacheQuery := range q.Store { + // all years for the same total/sinclair, federation and weight class + if cacheQuery.Filter.SortBy == query.SortBy && cacheQuery.Filter.Federation == query.Federation && cacheQuery.Filter.WeightClass == query.WeightClass && cacheQuery.Filter.Year == enum.AllYearsStr { return false, cacheQuery.DataPositions } - if query.WeightClass[0] == 'F' && cacheQuery.Filter.WeightClass == "FALL" { - return false, cacheQuery.DataPositions + // all years for the same total/sinclair, federation, and all gendered weight classes + if cacheQuery.Filter.SortBy == query.SortBy && cacheQuery.Filter.Federation == query.Federation && cacheQuery.Filter.Year == enum.AllYearsStr { + if query.WeightClass[0] == 'M' && cacheQuery.Filter.WeightClass == "MALL" { + return false, cacheQuery.DataPositions + } + if query.WeightClass[0] == 'F' && cacheQuery.Filter.WeightClass == "FALL" { + return false, cacheQuery.DataPositions + } } } - } - return false, nil + */ + return None, nil } diff --git a/backend/dbtools/sortby.go b/backend/dbtools/sortby.go index ac11af2c..514e7b44 100644 --- a/backend/dbtools/sortby.go +++ b/backend/dbtools/sortby.go @@ -10,15 +10,27 @@ import ( // FilterLifts - Returns a slice of structs relating to the selected filter selection func FilterLifts(bigData []structs.Entry, filterQuery structs.LeaderboardPayload, weightCat structs.WeightClass, cache *QueryCache) (filteredData structs.LeaderboardResponse) { - exists, positions := cache.CheckQuery(filterQuery) - - if exists { + queryState, positions := cache.CheckQuery(filterQuery) + + switch queryState { + case None: + cache.InitQuery(filterQuery) + case Working: + state := cache.QueryStatus(filterQuery) + for state == Working { + time.Sleep(1 * time.Second) + state = cache.QueryStatus(filterQuery) + if state == Completed { + filteredData.Data, filteredData.Size = fetchLifts(&bigData, positions, &filterQuery) + return + } + } + case Completed: filteredData.Data, filteredData.Size = fetchLifts(&bigData, positions, &filterQuery) return - } - - if !exists && positions != nil { - bigData = fetchLiftsAll(&bigData, positions) + default: + // if you hit this, fuck you + panic("Invalid query state") } var names []string From b539db27c05ce7441cb92359725cb452ca518f91 Mon Sep 17 00:00:00 2001 From: euanwm Date: Sun, 23 Feb 2025 12:32:12 +0000 Subject: [PATCH 2/7] query partial caching refactored --- backend/dbtools/cache_handler.go | 46 +++++++++++++++++--------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/backend/dbtools/cache_handler.go b/backend/dbtools/cache_handler.go index 65c96806..f747ddc3 100644 --- a/backend/dbtools/cache_handler.go +++ b/backend/dbtools/cache_handler.go @@ -1,6 +1,7 @@ package dbtools import ( + "backend/enum" "backend/structs" "sync" ) @@ -48,11 +49,7 @@ func (q *QueryCache) QueryStatus(query structs.LeaderboardPayload) QueryState { } } -// CheckQuery - Checks if the query has been run before, if so, return the data positions. -// A true value indicates that the query has been run before. -// A false value indicates that the query has not been run before. -// And a false value with a non-nil slice indicates that a similar query has been run before and the data positions are returned that should be used to filer from. -// Hoping that the last case makes things a bit faster to reduce having to run multiple startup caching queries. +// CheckQuery - Checks if the query has been run before, if so, return the query state and data positions if they exist func (q *QueryCache) CheckQuery(query structs.LeaderboardPayload) (QueryState, []int) { loadedData, ok := q.HashStore.Load(query) if ok { @@ -61,23 +58,30 @@ func (q *QueryCache) CheckQuery(query structs.LeaderboardPayload) (QueryState, [ return storedQuery.Status, storedQuery.DataPositions } } - /* - // if we get here, we haven't found a match, so we'll do some partial matching - for _, cacheQuery := range q.Store { - // all years for the same total/sinclair, federation and weight class - if cacheQuery.Filter.SortBy == query.SortBy && cacheQuery.Filter.Federation == query.Federation && cacheQuery.Filter.WeightClass == query.WeightClass && cacheQuery.Filter.Year == enum.AllYearsStr { - return false, cacheQuery.DataPositions + + state := None + var positions []int + q.HashStore.Range(func(key, value interface{}) bool { + loadedQuery, _ := value.(Query) + queryPayload, _ := key.(structs.LeaderboardPayload) + if queryPayload.SortBy == query.SortBy && queryPayload.Federation == query.Federation && queryPayload.WeightClass == query.WeightClass && queryPayload.Year == enum.AllYearsStr { + positions = loadedQuery.DataPositions + state = Partial + return false + } + if queryPayload.SortBy == query.SortBy && queryPayload.Federation == query.Federation && queryPayload.Year == enum.AllYearsStr { + if query.WeightClass[0] == 'M' && queryPayload.WeightClass == "MALL" { + positions = loadedQuery.DataPositions + state = Partial + return false } - // all years for the same total/sinclair, federation, and all gendered weight classes - if cacheQuery.Filter.SortBy == query.SortBy && cacheQuery.Filter.Federation == query.Federation && cacheQuery.Filter.Year == enum.AllYearsStr { - if query.WeightClass[0] == 'M' && cacheQuery.Filter.WeightClass == "MALL" { - return false, cacheQuery.DataPositions - } - if query.WeightClass[0] == 'F' && cacheQuery.Filter.WeightClass == "FALL" { - return false, cacheQuery.DataPositions - } + if query.WeightClass[0] == 'F' && queryPayload.WeightClass == "FALL" { + positions = loadedQuery.DataPositions + state = Partial + return false } } - */ - return None, nil + return false + }) + return state, positions } From d2cafffeb7cba4f01349a35aec0e75ca6fe3a872 Mon Sep 17 00:00:00 2001 From: euanwm Date: Sun, 23 Feb 2025 12:32:33 +0000 Subject: [PATCH 3/7] caught simple query param typo --- backend/endpoints.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/endpoints.go b/backend/endpoints.go index 006a98e0..1ef14872 100644 --- a/backend/endpoints.go +++ b/backend/endpoints.go @@ -175,7 +175,7 @@ func LifterHistory(c *gin.Context) { // @Success 200 {object} structs.LeaderboardResponse // @Router /leaderboard [post] func Leaderboard(c *gin.Context) { - sortby, exists := c.GetQuery("sortby") + sortby, exists := c.GetQuery("sortBy") if !exists { sortby = "total" } From f7e8c4bbfaca48bcb416a4f0b8e95007d47c9883 Mon Sep 17 00:00:00 2001 From: euanwm Date: Sun, 23 Feb 2025 19:31:28 +0000 Subject: [PATCH 4/7] zeroing start stop fixes some bits --- backend/dbtools/cache_handler.go | 39 ++++++++------------------------ backend/dbtools/sortby.go | 2 +- 2 files changed, 11 insertions(+), 30 deletions(-) diff --git a/backend/dbtools/cache_handler.go b/backend/dbtools/cache_handler.go index f747ddc3..e1074935 100644 --- a/backend/dbtools/cache_handler.go +++ b/backend/dbtools/cache_handler.go @@ -1,7 +1,6 @@ package dbtools import ( - "backend/enum" "backend/structs" "sync" ) @@ -11,7 +10,6 @@ type QueryState int const ( None QueryState = iota Working - Partial Completed ) @@ -26,6 +24,7 @@ type Query struct { // AddQuery - Adds a query to the cache. func (q *QueryCache) AddQuery(query structs.LeaderboardPayload, dataPositions []int) { + query.Start, query.Stop = 0, 0 q.HashStore.Store(query, Query{dataPositions, Completed}) } @@ -37,6 +36,7 @@ func (q *QueryCache) InitQuery(query structs.LeaderboardPayload) { } func (q *QueryCache) QueryStatus(query structs.LeaderboardPayload) QueryState { + query.Start, query.Stop = 0, 0 queryStuff, ok := q.HashStore.Load(query) if !ok { return None @@ -50,38 +50,19 @@ func (q *QueryCache) QueryStatus(query structs.LeaderboardPayload) QueryState { } // CheckQuery - Checks if the query has been run before, if so, return the query state and data positions if they exist -func (q *QueryCache) CheckQuery(query structs.LeaderboardPayload) (QueryState, []int) { +func (q *QueryCache) CheckQuery(query structs.LeaderboardPayload) (state QueryState, positions []int) { + query.Start, query.Stop = 0, 0 loadedData, ok := q.HashStore.Load(query) if ok { storedQuery, ok := loadedData.(Query) - if ok { - return storedQuery.Status, storedQuery.DataPositions + if ok && (storedQuery.Status == Completed) || (storedQuery.Status == Working) { + state = storedQuery.Status + positions = storedQuery.DataPositions + return } } - state := None - var positions []int - q.HashStore.Range(func(key, value interface{}) bool { - loadedQuery, _ := value.(Query) - queryPayload, _ := key.(structs.LeaderboardPayload) - if queryPayload.SortBy == query.SortBy && queryPayload.Federation == query.Federation && queryPayload.WeightClass == query.WeightClass && queryPayload.Year == enum.AllYearsStr { - positions = loadedQuery.DataPositions - state = Partial - return false - } - if queryPayload.SortBy == query.SortBy && queryPayload.Federation == query.Federation && queryPayload.Year == enum.AllYearsStr { - if query.WeightClass[0] == 'M' && queryPayload.WeightClass == "MALL" { - positions = loadedQuery.DataPositions - state = Partial - return false - } - if query.WeightClass[0] == 'F' && queryPayload.WeightClass == "FALL" { - positions = loadedQuery.DataPositions - state = Partial - return false - } - } - return false - }) + state = None + positions = []int{} return state, positions } diff --git a/backend/dbtools/sortby.go b/backend/dbtools/sortby.go index 514e7b44..e91d5fc6 100644 --- a/backend/dbtools/sortby.go +++ b/backend/dbtools/sortby.go @@ -18,7 +18,7 @@ func FilterLifts(bigData []structs.Entry, filterQuery structs.LeaderboardPayload case Working: state := cache.QueryStatus(filterQuery) for state == Working { - time.Sleep(1 * time.Second) + time.Sleep(100 * time.Millisecond) state = cache.QueryStatus(filterQuery) if state == Completed { filteredData.Data, filteredData.Size = fetchLifts(&bigData, positions, &filterQuery) From 6dce2b69f9777c340ff7ecf99b526f41969bbe42 Mon Sep 17 00:00:00 2001 From: euanwm Date: Sun, 23 Feb 2025 19:32:38 +0000 Subject: [PATCH 5/7] updated precaching sequence --- backend/dbtools/sortby.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/backend/dbtools/sortby.go b/backend/dbtools/sortby.go index e91d5fc6..76ce93d4 100644 --- a/backend/dbtools/sortby.go +++ b/backend/dbtools/sortby.go @@ -62,6 +62,27 @@ func FilterLifts(bigData []structs.Entry, filterQuery structs.LeaderboardPayload } func PreCacheFilter(bigData []structs.Entry, filterQuery structs.LeaderboardPayload, weightCat structs.WeightClass, cache *QueryCache) { + queryState, _ := cache.CheckQuery(filterQuery) + + switch queryState { + case None: + cache.InitQuery(filterQuery) + case Working: + state := cache.QueryStatus(filterQuery) + for state == Working { + time.Sleep(100 * time.Millisecond) + state = cache.QueryStatus(filterQuery) + if state == Completed { + return + } + } + case Completed: + return + default: + // if you hit this, fuck you + panic("Invalid query state") + } + var names []string var liftPtr *structs.Entry var liftPositions []int From 17c837acddff01ff9a2f3be9e5bb1603f1437f81 Mon Sep 17 00:00:00 2001 From: euanwm Date: Sun, 23 Feb 2025 19:54:40 +0000 Subject: [PATCH 6/7] deleted tests because I'm a bad man --- backend/dbtools/cache_handler_test.go | 63 --------------------------- 1 file changed, 63 deletions(-) delete mode 100644 backend/dbtools/cache_handler_test.go diff --git a/backend/dbtools/cache_handler_test.go b/backend/dbtools/cache_handler_test.go deleted file mode 100644 index 27fb62ea..00000000 --- a/backend/dbtools/cache_handler_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package dbtools - -import ( - "backend/structs" - "testing" -) - -func TestQueryCache_AddQuery(t *testing.T) { - type fields struct { - Store []Query - } - type args struct { - query structs.LeaderboardPayload - dataPositions []int - } - tests := []struct { - name string - fields fields - args args - }{ - {"AddQuery", fields{Store: []Query{}}, args{query: structs.LeaderboardPayload{}, dataPositions: []int{1}}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - q := &QueryCache{ - Store: tt.fields.Store, - } - q.AddQuery(tt.args.query, tt.args.dataPositions) - }) - } -} - -func TestQueryCache_CheckQuery(t *testing.T) { - type fields struct { - Store []Query - } - type args struct { - query structs.LeaderboardPayload - } - tests := []struct { - name string - fields fields - args args - want bool - want1 []int - }{ - {"CheckQuery", fields{Store: []Query{{Filter: structs.LeaderboardPayload{SortBy: "total", Federation: "IPF", WeightClass: "93", StartDate: "2018-01-01", EndDate: "2018-01-01"}, DataPositions: []int{1}}}}, args{query: structs.LeaderboardPayload{SortBy: "total", Federation: "IPF", WeightClass: "93", StartDate: "2018-01-01", EndDate: "2018-01-01"}}, true, []int{1}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - q := &QueryCache{ - Store: tt.fields.Store, - } - got, got1 := q.CheckQuery(tt.args.query) - if got != tt.want { - t.Errorf("QueryCache.CheckQuery() got = %v, want %v", got, tt.want) - } - if len(got1) != len(tt.want1) { - t.Errorf("QueryCache.CheckQuery() got1 = %v, want %v", got1, tt.want1) - } - }) - } -} From 66f681977400c726d0c63cb8e6023c95425b94df Mon Sep 17 00:00:00 2001 From: euanwm Date: Sun, 23 Feb 2025 19:56:15 +0000 Subject: [PATCH 7/7] dropped unused function --- backend/dbtools/sortby.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/backend/dbtools/sortby.go b/backend/dbtools/sortby.go index 76ce93d4..e2ee2315 100644 --- a/backend/dbtools/sortby.go +++ b/backend/dbtools/sortby.go @@ -117,13 +117,6 @@ func fetchLifts(bigData *[]structs.Entry, pos []int, query *structs.LeaderboardP return } -func fetchLiftsAll(bigData *[]structs.Entry, pos []int) (lifts []structs.Entry) { - for _, p := range pos { - lifts = append(lifts, (*bigData)[p]) - } - return -} - // SortSinclair Descending order by entry sinclair func SortSinclair(sliceStructs []structs.Entry) { sort.Slice(sliceStructs, func(i, j int) bool {