Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reports caching task count #1132

Merged
merged 5 commits into from
Jan 2, 2025
Merged

Reports caching task count #1132

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7cb99c9
Added Task Analytics Cron Setup
Shanks0465 Dec 21, 2024
3b5e9df
Added Task Analytics Caching
Shanks0465 Dec 21, 2024
0aed468
Added On Start Trigger for Task Count
Shanks0465 Dec 21, 2024
2c096f2
Added Workspace Task Analytics Cron
Shanks0465 Dec 27, 2024
abdc6e6
Added OCRSegmentCategorizationEditing Task Count
Shanks0465 Jan 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 57 additions & 45 deletions backend/organizations/views.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from rest_framework import status
from tasks.models import (
Task,
Statistic,
ANNOTATOR_ANNOTATION,
REVIEWER_ANNOTATION,
SUPER_CHECKER_ANNOTATION,
Expand Down Expand Up @@ -461,11 +462,11 @@ def quality_reports(self, request, pk=None):
participation_type = (
"Full Time"
if participation_type == 1
else "Part Time"
if participation_type == 2
else "Contract Basis"
if participation_type == 4
else "N/A"
else (
"Part Time"
if participation_type == 2
else "Contract Basis" if participation_type == 4 else "N/A"
)
)
role = get_role_name(annotator.role)
user_id = annotator.id
Expand Down Expand Up @@ -779,11 +780,11 @@ def user_analytics(self, request, pk=None):
participation_type = (
"Full Time"
if participation_type == 1
else "Part Time"
if participation_type == 2
else "Contract Basis"
if participation_type == 4
else "N/A"
else (
"Part Time"
if participation_type == 2
else "Contract Basis" if participation_type == 4 else "N/A"
)
)
role = get_role_name(annotator.role)
user_id = annotator.id
Expand Down Expand Up @@ -935,9 +936,9 @@ def iter_items(items, pseudo_buffer):
status=status.HTTP_200_OK,
content_type="text/csv",
)
response[
"Content-Disposition"
] = f'attachment; filename="{organization.title}_user_analytics.csv"'
response["Content-Disposition"] = (
f'attachment; filename="{organization.title}_user_analytics.csv"'
)
return response

return Response(data=final_result, status=status.HTTP_200_OK)
Expand Down Expand Up @@ -2993,29 +2994,30 @@ def cumulative_tasks_count(self, request, pk=None):
}

else:
reviewer_task_count = (
reviewer_tasks.count()
+ reviewer_tasks_exported.count()
+ supercheck_tasks_exported.count()
)

annotation_tasks_count = (
annotation_tasks.count()
+ annotation_tasks_exported.count()
+ reviewer_tasks_exported.count()
+ supercheck_tasks_exported.count()
)

supercheck_tasks_count = (
supercheck_tasks.count() + supercheck_tasks_exported.count()
)

result = {
"language": lang,
"ann_cumulative_tasks_count": annotation_tasks_count,
"rew_cumulative_tasks_count": reviewer_task_count,
"sup_cumulative_tasks_count": supercheck_tasks_count,
}
# reviewer_task_count = (
# reviewer_tasks.count()
# + reviewer_tasks_exported.count()
# + supercheck_tasks_exported.count()
# )

# annotation_tasks_count = (
# annotation_tasks.count()
# + annotation_tasks_exported.count()
# + reviewer_tasks_exported.count()
# + supercheck_tasks_exported.count()
# )

# supercheck_tasks_count = (
# supercheck_tasks.count() + supercheck_tasks_exported.count()
# )

# result = {
# "language": lang,
# "ann_cumulative_tasks_count": annotation_tasks_count,
# "rew_cumulative_tasks_count": reviewer_task_count,
# "sup_cumulative_tasks_count": supercheck_tasks_count,
# }
result = {}

if lang == None or lang == "":
other_lang.append(result)
Expand All @@ -3037,9 +3039,10 @@ def cumulative_tasks_count(self, request, pk=None):
rev_sentance_count = 0
for dat in other_lang:
if metainfo != True:
ann_task_count += dat["ann_cumulative_tasks_count"]
rew_task_count += dat["rew_cumulative_tasks_count"]
sup_task_count += dat["sup_cumulative_tasks_count"]
# ann_task_count += dat["ann_cumulative_tasks_count"]
# rew_task_count += dat["rew_cumulative_tasks_count"]
# sup_task_count += dat["sup_cumulative_tasks_count"]
pass
else:
if project_type in get_audio_project_types():
ann_aud_dur += convert_hours_to_seconds(
Expand Down Expand Up @@ -3078,12 +3081,13 @@ def cumulative_tasks_count(self, request, pk=None):

if len(other_lang) > 0:
if metainfo != True:
other_language = {
"language": "Others",
"ann_cumulative_tasks_count": ann_task_count,
"rew_cumulative_tasks_count": rew_task_count,
"sup_cumulative_tasks_count": sup_task_count,
}
# other_language = {
# "language": "Others",
# "ann_cumulative_tasks_count": ann_task_count,
# "rew_cumulative_tasks_count": rew_task_count,
# "sup_cumulative_tasks_count": sup_task_count,
# }
other_language = {}
else:
if project_type in get_audio_project_types():
other_language = {
Expand Down Expand Up @@ -3155,4 +3159,12 @@ def cumulative_tasks_count(self, request, pk=None):
pass
else:
final_result_for_all_types[project_type] = final_result
if metainfo != True:

task_counts = list(
Statistic.objects.filter(stat_type="task_count", org_id=organization.id)
)[0].result

for pjt_type in project_types:
final_result_for_all_types[pjt_type] = task_counts[pjt_type]
return Response(final_result_for_all_types)
14 changes: 14 additions & 0 deletions backend/shoonya_backend/celery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from __future__ import absolute_import, unicode_literals
from datetime import timedelta
from celery.schedules import crontab
from celery.signals import worker_ready
import os

from celery import Celery
Expand Down Expand Up @@ -38,8 +39,21 @@
"task": "check_size",
"schedule": crontab(minute=0, hour=0), # every mid night
},
"fetchTaskCounts": {"task": "fetchTaskCounts", "schedule": crontab(minute="*/10")},
"fetchWorkspaceTaskCounts": {
"task": "fetchWorkspaceTaskCounts",
"schedule": crontab(minute="*/10"),
},
}


@worker_ready.connect
def at_start(sender, **k):
with sender.app.connection() as conn:
sender.app.send_task("fetchTaskCounts", connection=conn)
sender.app.send_task("fetchWorkspaceTaskCounts", connection=conn)


# Celery Task related settings
celery_app.autodiscover_tasks()

Expand Down
10 changes: 10 additions & 0 deletions backend/tasks/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,16 @@ class Meta:
)


class Statistic(models.Model):
stat_type = models.CharField(max_length=255)
org_id = models.IntegerField()
result = models.JSONField()
updated_at = models.DateTimeField(auto_now=True)

class Meta:
unique_together = ("stat_type", "org_id")


class Prediction(models.Model):
"""ML predictions"""

Expand Down
Loading
Loading