diff --git a/bedrock/base/geo.py b/bedrock/base/geo.py index b3a99389afc..f439f564c59 100644 --- a/bedrock/base/geo.py +++ b/bedrock/base/geo.py @@ -3,12 +3,20 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. from django.conf import settings +from django.core.cache import cache from product_details import product_details def valid_country_code(country): - codes = product_details.get_regions("en-US").keys() + _key = f"valid_country_codes_for_{country}" + + codes = cache.get(_key) + + if not codes: + codes = product_details.get_regions("en-US").keys() + cache.set(_key, codes, timeout=settings.CACHE_TIME_MED) + if country and country.lower() in codes: return country.upper() diff --git a/bedrock/careers/models.py b/bedrock/careers/models.py index 2f9898470f5..8e4d847d074 100644 --- a/bedrock/careers/models.py +++ b/bedrock/careers/models.py @@ -5,6 +5,8 @@ from datetime import datetime from itertools import chain +from django.conf import settings +from django.core.cache import cache from django.db import models from django.urls import reverse @@ -37,28 +39,52 @@ def __str__(self): def get_absolute_url(self): return reverse("careers.position", kwargs={"source": self.source, "job_id": self.job_id}) + @classmethod + def _get_cache_key(cls, name): + return f"careers_position__{name}" + @property def location_list(self): - return sorted(self.location.split(",")) + _key = self._get_cache_key("location_list") + location_list = cache.get(_key) + if location_list is None: + location_list = sorted(self.location.split(",")) + cache.set(_key, location_list, settings.CACHE_TIME_LONG) + return location_list @classmethod def position_types(cls): - return sorted(set(cls.objects.values_list("position_type", flat=True))) + _key = cls._get_cache_key("position_types") + position_types = cache.get(_key) + if position_types is None: + position_types = sorted(set(cls.objects.values_list("position_type", flat=True))) + cache.set(_key, position_types, settings.CACHE_TIME_LONG) + return position_types @classmethod def locations(cls): - return sorted( - { - location.strip() - for location in chain( - *[locations.split(",") for locations in cls.objects.exclude(job_locations="Remote").values_list("job_locations", flat=True)] - ) - } - ) + _key = cls._get_cache_key("locations") + locations = cache.get(_key) + if locations is None: + locations = sorted( + { + location.strip() + for location in chain( + *[locations.split(",") for locations in cls.objects.exclude(job_locations="Remote").values_list("job_locations", flat=True)] + ) + } + ) + cache.set(_key, locations, settings.CACHE_TIME_LONG) + return locations @classmethod def categories(cls): - return sorted(set(cls.objects.values_list("department", flat=True))) + _key = cls._get_cache_key("categories") + categories = cache.get(_key) + if categories is None: + categories = sorted(set(cls.objects.values_list("department", flat=True))) + cache.set(_key, categories, settings.CACHE_TIME_LONG) + return categories @property def cover(self): diff --git a/bedrock/careers/tests/test_forms.py b/bedrock/careers/tests/test_forms.py index 7cd719f157c..dc4c81ef2da 100644 --- a/bedrock/careers/tests/test_forms.py +++ b/bedrock/careers/tests/test_forms.py @@ -2,12 +2,18 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. + +from django.core.cache import cache + from bedrock.careers.forms import PositionFilterForm from bedrock.careers.tests import PositionFactory from bedrock.mozorg.tests import TestCase class PositionFilterFormTests(TestCase): + def setUp(self): + cache.clear() + def test_dynamic_position_type_choices(self): """ The choices for the position_type field should be dynamically diff --git a/bedrock/careers/tests/test_models.py b/bedrock/careers/tests/test_models.py index b1a68cdcf5c..68f4894c30a 100644 --- a/bedrock/careers/tests/test_models.py +++ b/bedrock/careers/tests/test_models.py @@ -2,12 +2,17 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +from django.core.cache import cache + from bedrock.careers.models import Position from bedrock.careers.tests import PositionFactory from bedrock.mozorg.tests import TestCase class TestPositionModel(TestCase): + def setUp(self): + cache.clear() + def test_location_list(self): PositionFactory(location="San Francisco,Portland") pos = Position.objects.get() diff --git a/bedrock/careers/tests/test_utils.py b/bedrock/careers/tests/test_utils.py index 022a56d8020..232af701d03 100644 --- a/bedrock/careers/tests/test_utils.py +++ b/bedrock/careers/tests/test_utils.py @@ -2,6 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +from django.core.cache import cache + from bedrock.careers.tests import PositionFactory from bedrock.careers.utils import generate_position_meta_description from bedrock.mozorg.tests import TestCase @@ -9,6 +11,7 @@ class GeneratePositionMetaDescriptionTests(TestCase): def setUp(self): + cache.clear() self.position = PositionFactory(title="Bowler", position_type="Full time", location="Los Angeles,Ralphs") def test_position_type_consonant_beginning(self): diff --git a/bedrock/careers/views.py b/bedrock/careers/views.py index 9c2afc15a35..b5aafc5301a 100644 --- a/bedrock/careers/views.py +++ b/bedrock/careers/views.py @@ -2,6 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +from django.conf import settings +from django.core.cache import cache from django.http.response import Http404 from django.shortcuts import get_object_or_404 from django.views.generic import DetailView, ListView @@ -62,10 +64,17 @@ class BenefitsView(L10nTemplateView): class PositionListView(LangFilesMixin, RequireSafeMixin, ListView): - queryset = Position.objects.exclude(job_locations="Remote") template_name = "careers/listings.html" context_object_name = "positions" + def get_queryset(self): + _key = "careers_position_listing_qs" + qs = cache.get(_key) + if qs is None: + qs = Position.objects.exclude(job_locations="Remote") + cache.set(_key, qs, settings.CACHE_TIME_SHORT) + return qs + def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["form"] = PositionFilterForm() diff --git a/bedrock/newsletter/utils.py b/bedrock/newsletter/utils.py index 11d4f1c088b..77f6c02fc2e 100644 --- a/bedrock/newsletter/utils.py +++ b/bedrock/newsletter/utils.py @@ -2,6 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +from django.conf import settings +from django.core.cache import cache + import basket from bedrock.newsletter.models import Newsletter @@ -12,7 +15,12 @@ def get_newsletters(): Keys are the internal keys we use to designate newsletters to basket. Values are dictionaries with the remaining newsletter information. """ - return Newsletter.objects.serialize() + _key = "serialized_newsletters" + serialized_newsletters = cache.get(_key) + if serialized_newsletters is None: + serialized_newsletters = Newsletter.objects.serialize() + cache.set(_key, serialized_newsletters, timeout=settings.CACHE_TIME_LONG) + return serialized_newsletters def get_languages_for_newsletters(newsletters=None): diff --git a/bedrock/settings/base.py b/bedrock/settings/base.py index 288e325eb32..510c1399b61 100644 --- a/bedrock/settings/base.py +++ b/bedrock/settings/base.py @@ -86,11 +86,16 @@ def data_path(*args): "image_renditions": {"URL": f"{REDIS_URL}/0"}, } +CACHE_TIME_SHORT = 60 * 10 # 10 mins +CACHE_TIME_MED = 60 * 60 # 1 hour +CACHE_TIME_LONG = 60 * 60 * 6 # 6 hours + + CACHES = { "default": { "BACKEND": "bedrock.base.cache.SimpleDictCache", "LOCATION": "default", - "TIMEOUT": 600, + "TIMEOUT": CACHE_TIME_SHORT, "OPTIONS": { "MAX_ENTRIES": 5000, "CULL_FREQUENCY": 4, # 1/4 entries deleted if max reached @@ -2443,3 +2448,11 @@ def lazy_wagtail_langs(): # Useful when customising the Wagtail admin # when enabled, will be visible on cms-admin/styleguide INSTALLED_APPS.append("wagtail.contrib.styleguide") + +# Django-silk for performance profiling +if ENABLE_DJANGO_SILK := config("ENABLE_DJANGO_SILK", default="False", parser=bool): + print("Django-Silk profiling enabled - go to http://localhost:8000/silk/ to view metrics") + INSTALLED_APPS.append("silk") + MIDDLEWARE.insert(0, "silk.middleware.SilkyMiddleware") + SUPPORTED_NONLOCALES.append("silk") + SILKY_PYTHON_PROFILER = config("SILKY_PYTHON_PROFILER", default="False", parser=bool) diff --git a/bedrock/urls.py b/bedrock/urls.py index a9b73d2548a..cee73330edf 100644 --- a/bedrock/urls.py +++ b/bedrock/urls.py @@ -72,6 +72,9 @@ path("_internal_draft_preview/", include(wagtaildraftsharing_urls)), # ONLY available in CMS mode ) +if settings.ENABLE_DJANGO_SILK: + urlpatterns += [path("silk/", include("silk.urls", namespace="silk"))] + if settings.DEFAULT_FILE_STORAGE == "django.core.files.storage.FileSystemStorage": # Serve media files from Django itself - production won't use this from django.urls import re_path diff --git a/profiling/hit_popular_pages.py b/profiling/hit_popular_pages.py new file mode 100644 index 00000000000..d12e362a405 --- /dev/null +++ b/profiling/hit_popular_pages.py @@ -0,0 +1,96 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Request a selection of pages that are populat on www.m.o from your local +runserver, so that django-silk can capture performance info on them. + +Usage: + + 1. In your .env set ENABLE_DJANGO_SILK=True + 2. Start your runserver on port 8000 + 3. python profiling/hit_popular_pages.py + 3. View results at http://localhost:8000/silk/ + +""" + +import sys +import time + +import requests + +paths = [ + "/en-US/firefox/126.0/whatsnew/", + "/en-US/firefox/", + "/en-US/firefox/windows/", + "/en-US/firefox/new/?reason=manual-update", + "/en-US/firefox/download/thanks/", + "/en-US/firefox/new/?reason=outdated", + "/en-US/firefox/features/", + "/en-US/firefox/all/", + "/en-US/firefox/welcome/18/", + "/en-US/", + "/en-US/firefox/installer-help/?channel=release&installer_lang=en-US", + "/en-US/firefox/download/thanks/?s=direct", + "/en-US/firefox/welcome/19/", + "/en-US/firefox/enterprise/?reason=manual-update", + "/en-US/products/vpn/", + "/en-US/firefox/browsers/windows-64-bit/", + "/en-US/firefox/mac/", + "/en-US/about/", + "/en-US/firefox/android/124.0/releasenotes/", + "/en-US/firefox/browsers/mobile/get-app/", + "/en-US/firefox/browsers/", + "/en-US/firefox/nightly/firstrun/", + "/en-US/firefox/developer/", + "/en-US/account/", + "/en-US/contribute/", + "/en-US/firefox/browsers/mobile/android/", + "/en-US/privacy/archive/firefox-fire-tv/2023-06/", + "/en-US/firefox/121.0/system-requirements/", + "/en-US/firefox/browsers/mobile/", + "/en-US/firefox/releases/", + "/en-US/MPL/", + "/en-US/firefox/enterprise/", + "/en-US/security/advisories/", + "/en-US/firefox/browsers/what-is-a-browser/", + "/en-US/firefox/channel/desktop/?reason=manual-update", + "/en-US/firefox/pocket/", + "/en-US/firefox/channel/desktop/", + "/en-US/firefox/welcome/17b/", + "/en-US/firefox/welcome/17c/", + "/en-US/firefox/welcome/17a/", + "/en-US/firefox/set-as-default/thanks/", + "/en-US/careers/listings/", + "/en-US/firefox/browsers/chromebook/", + "/en-US/firefox/nothing-personal/", + "/en-US/newsletter/existing/", + "/en-US/about/legal/terms/firefox/", + "/en-US/firefox/linux/", + "/en-US/firefox/browsers/mobile/focus/", + "/en-US/products/vpn/download/", + "/en-US/about/manifesto/", + "/en-US/stories/joy-of-color/", + "/en-US/contact/", + "/en-US/about/legal/defend-mozilla-trademarks/", +] + + +def _log(*args): + sys.stdout.write("\n".join(args)) + + +def hit_pages(paths, times=3): + _base_url = "http://localhost:8000" + + for path in paths: + for _ in range(times): + time.sleep(0.5) + url = f"{_base_url}{path}" + requests.get(url) + + _log("All done") + + +if __name__ == "__main__": + hit_pages(paths) diff --git a/requirements/dev.in b/requirements/dev.in index fde79ac9201..12fee16abc3 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -2,6 +2,7 @@ bpython==0.24 braceexpand==0.1.7 +django-silk==5.3.1 factory-boy==3.3.1 freezegun==1.5.1 markdown-it-py>=2.2.0 diff --git a/requirements/dev.txt b/requirements/dev.txt index 9c3ae095347..22dbe610eb2 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -33,6 +33,10 @@ attrs==24.3.0 \ # outcome # referencing # trio +autopep8==2.3.1 \ + --hash=sha256:8d6c87eba648fdcfc83e29b788910b8643171c395d9c4bcf115ece035b9c9dda \ + --hash=sha256:a203fe0fcad7939987422140ab17a930f684763bf7335bdb6709991dd7ef6c2d + # via django-silk babel==2.16.0 \ --hash=sha256:368b5b98b37c06b7daf6696391c3240c938b37767d4584413e8438c5c435fa8b \ --hash=sha256:d1f3554ca26605fe173f3de0c65f750f5a42f924499bf134de6423582298e316 @@ -493,6 +497,7 @@ django==4.2.17 \ # django-permissionedforms # django-rq # django-rq-email-backend + # django-silk # django-storages # django-taggit # django-treebeard @@ -573,6 +578,10 @@ django-rq-email-backend==2.0.0 \ --hash=sha256:402ced0d8078a856a684be37206fbfb0a6056c5bfe8a753dd7eec7fcaccca224 \ --hash=sha256:4e8a6c6f492f9e78711e6b563c96f7b8feaf17c8737767a30fc68b9833e3b82f # via -r requirements/prod.txt +django-silk==5.3.1 \ + --hash=sha256:7834580fabea5d9e8a32eabb0d9cb061cc37f0ec057d2933b4da761a53ae1bed \ + --hash=sha256:aa4ae73a90fcbd5159a810f81e15a0ec010619beab37c82957eb4012fd0016f0 + # via -r requirements/dev.in django-storages[google]==1.14.4 \ --hash=sha256:69aca94d26e6714d14ad63f33d13619e697508ee33ede184e462ed766dc2a73f \ --hash=sha256:d61930acb4a25e3aebebc6addaf946a3b1df31c803a6bf1af2f31c9047febaa3 @@ -800,6 +809,10 @@ granian==1.6.3 \ --hash=sha256:edf6a6cb8318b11bf1bb90c3c1ad5af426b283849b28749e80d26e3aba3f8ad8 \ --hash=sha256:f4ea547a9cb850eaa6cd448374051c657eaa503bb720d2b0939945193da1f548 # via -r requirements/prod.txt +gprof2dot==2024.6.6 \ + --hash=sha256:45b14ad7ce64e299c8f526881007b9eb2c6b75505d5613e96e66ee4d5ab33696 \ + --hash=sha256:fa1420c60025a9eb7734f65225b4da02a10fc6dd741b37fa129bc6b41951e5ab + # via django-silk greenlet==3.1.1 \ --hash=sha256:0153404a4bb921f0ff1abeb5ce8a5131da56b953eda6e14b88dc6bbc04d2049e \ --hash=sha256:03a088b9de532cbfe2ba2034b2b85e82df37874681e8c470d6fb2f8c04d7e4b7 \ @@ -1596,6 +1609,10 @@ pyasn1-modules==0.4.1 \ # via # -r requirements/prod.txt # google-auth +pycodestyle==2.12.1 \ + --hash=sha256:46f0fb92069a7c28ab7bb558f05bfc0110dac69a0cd23c61ea0040283a9d78b3 \ + --hash=sha256:6838eae08bbce4f6accd5d5572075c63626a15ee3e6f842df996bf62f6d73521 + # via autopep8 pycparser==2.22 \ --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \ --hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc @@ -2139,6 +2156,7 @@ sqlparse==0.5.3 \ # via # -r requirements/prod.txt # django + # django-silk supervisor==4.2.5 \ --hash=sha256:2ecaede32fc25af814696374b79e42644ecaba5c09494c51016ffda9602d0f08 \ --hash=sha256:34761bae1a23c58192281a5115fb07fbf22c9b0133c08166beffc70fed3ebc12