From 08b37701d817332fb7494ae3fdf18ffdcc3e283e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Choutri=20de=20Tarl=C3=A9?= Date: Thu, 2 Jan 2025 14:51:23 +0100 Subject: [PATCH] Add prometheus counter for package imports (#811) --- app/cli/Main.hs | 16 +++- app/server/Main.hs | 21 ++--- cabal.project.repositories | 14 +-- changelog.d/811 | 2 + environment.sh | 2 +- flora.cabal | 3 + src/core/Flora/Environment.hs | 61 ++----------- src/core/Flora/Environment/Env.hs | 57 ++++++++++++ src/core/Flora/Import/Package/Bulk.hs | 32 +++++-- src/core/Flora/Model/Job.hs | 3 +- src/core/Flora/Model/PackageIndex/Query.hs | 10 ++- src/core/Flora/Monitoring.hs | 88 +++++++++++++++++++ src/jobs-worker/FloraJobs/Runner.hs | 61 +++++++------ src/jobs-worker/FloraJobs/Scheduler.hs | 48 +++++----- src/jobs-worker/FloraJobs/Types.hs | 20 ++++- src/web/FloraWeb/Common/Auth.hs | 12 +-- src/web/FloraWeb/Common/Tracing.hs | 2 +- src/web/FloraWeb/LiveReload.hs | 6 +- src/web/FloraWeb/Pages/Server/Admin.hs | 10 ++- src/web/FloraWeb/Pages/Server/Categories.hs | 8 +- src/web/FloraWeb/Pages/Server/Packages.hs | 2 +- src/web/FloraWeb/Pages/Server/Settings.hs | 8 +- src/web/FloraWeb/Pages/Templates.hs | 2 +- src/web/FloraWeb/Pages/Templates/Error.hs | 2 +- src/web/FloraWeb/Pages/Templates/Packages.hs | 2 +- .../FloraWeb/Pages/Templates/Screens/Home.hs | 2 +- src/web/FloraWeb/Pages/Templates/Types.hs | 8 +- src/web/FloraWeb/Server.hs | 16 ++-- src/web/FloraWeb/Session.hs | 2 +- src/web/FloraWeb/Types.hs | 4 +- test/Flora/ImportSpec.hs | 1 + test/Flora/TestUtils.hs | 17 +++- 32 files changed, 361 insertions(+), 181 deletions(-) create mode 100644 changelog.d/811 create mode 100644 src/core/Flora/Environment/Env.hs create mode 100644 src/core/Flora/Monitoring.hs diff --git a/app/cli/Main.hs b/app/cli/Main.hs index 96e0ca6e..711510b1 100644 --- a/app/cli/Main.hs +++ b/app/cli/Main.hs @@ -19,6 +19,8 @@ import Effectful.FileSystem import Effectful.Log (Log, runLog) import Effectful.Poolboy import Effectful.PostgreSQL.Transact.Effect +import Effectful.Reader.Static (Reader) +import Effectful.Reader.Static qualified as Reader import Effectful.State.Static.Shared (State) import Effectful.State.Static.Shared qualified as State import Effectful.Time (Time, runTime) @@ -26,6 +28,7 @@ import Effectful.Trace (Trace) import Effectful.Trace qualified as Trace import GHC.Conc import GHC.Generics (Generic) +import GHC.Records import Log qualified import Log.Backend.StandardOutput qualified as Log import Monitor.Tracing.Zipkin (Zipkin (..)) @@ -36,7 +39,8 @@ import System.FilePath (()) import Advisories.Import (importAdvisories) import Advisories.Import.Error (AdvisoryImportError) -import Flora.Environment +import Flora.Environment (getFloraEnv) +import Flora.Environment.Env import Flora.Import.Categories (importCategories) import Flora.Import.Package.Bulk (importAllFilesInRelativeDirectory, importFromIndex) import Flora.Model.BlobIndex.Update qualified as Update @@ -108,6 +112,7 @@ main = Log.withStdOutLogger $ \logger -> do ) . runFileSystem . runLog "flora-cli" logger Log.LogTrace + . Reader.runReader env $ runOptions cliArgs case result of Right _ -> pure () @@ -196,6 +201,9 @@ runOptions , Poolboy :> es , Error (NonEmpty AdvisoryImportError) :> es , Trace :> es + , HasField "metrics" r Metrics + , HasField "mltp" r MLTP + , Reader r :> es ) => Options -> Eff es () @@ -245,6 +253,9 @@ importFolderOfCabalFiles , DB :> es , IOE :> es , State (Set (Namespace, PackageName, Version)) :> es + , HasField "metrics" r Metrics + , HasField "mltp" r MLTP + , Reader r :> es ) => FilePath -> Text @@ -264,6 +275,9 @@ importIndex , DB :> es , IOE :> es , State (Set (Namespace, PackageName, Version)) :> es + , HasField "metrics" r Metrics + , HasField "mltp" r MLTP + , Reader r :> es ) => FilePath -> Text diff --git a/app/server/Main.hs b/app/server/Main.hs index c9c3b63b..04297315 100644 --- a/app/server/Main.hs +++ b/app/server/Main.hs @@ -4,12 +4,12 @@ module Main where -import Control.Monad (forM_, unless, void) +import Control.Monad (forM_, unless) import Data.Function ((&)) import Data.List qualified as List -import Data.Pool (Pool) import Data.Set qualified as Set import Data.Text (Text) +import Data.Text qualified as Text import Data.Vector (Vector) import Data.Vector qualified as Vector import Database.PostgreSQL.Entity @@ -24,12 +24,11 @@ import Log qualified import System.Exit import System.IO -import Data.Text qualified as Text -import Database.PostgreSQL.Simple qualified as PG -import Flora.Environment (FloraEnv (..), MLTP (..), getFloraEnv) +import Flora.Environment (getFloraEnv) +import Flora.Environment.Env (FloraEnv (..), MLTP (..)) import Flora.Logging qualified as Logging import Flora.Model.PackageIndex.Types -import FloraJobs.Scheduler (checkIfIndexRefreshJobIsPlanned, scheduleRefreshIndexes) +import FloraJobs.Scheduler (checkIfIndexRefreshJobIsPlanned) import FloraWeb.Server main :: IO () @@ -47,17 +46,9 @@ main = do Log.LogTrace $ do checkRepositoriesAreConfigured - checkIndexRefreshScheduling env.pool + checkIfIndexRefreshJobIsPlanned env.pool runFlora -checkIndexRefreshScheduling :: (DB :> es, Log :> es, IOE :> es) => Pool PG.Connection -> Eff es () -checkIndexRefreshScheduling pool = do - indexRefreshIsPlanned <- - checkIfIndexRefreshJobIsPlanned - unless indexRefreshIsPlanned $ do - Log.logInfo_ "Scheduling index refresh" - void $ liftIO $ scheduleRefreshIndexes pool - checkRepositoriesAreConfigured :: (DB :> es, Log :> es, IOE :> es) => Eff es () checkRepositoriesAreConfigured = do let expectedRepositories = Set.fromList ["hackage", "cardano", "horizon"] diff --git a/cabal.project.repositories b/cabal.project.repositories index d93d1265..8ed16b78 100644 --- a/cabal.project.repositories +++ b/cabal.project.repositories @@ -9,10 +9,10 @@ repository cardano c00aae8461a256275598500ea0e187588c35a5d5d7454fb57eac18d9edb86a56 d4a35cd3121aa00d18544bb0ac01c3e1691d618f462c46129271bccf39f7e8ee -repository horizon - url: https://packages.horizon-haskell.net/ - secure: True - root-keys: - 272e995c7a74de109518100e1422193fe5e5971e52c92b98147c9355b47d7bb6 - ea5c1bc0944dabe64d9d68c6813a8141d747cda042b870576d7af63a2326c31b - eb47482ddf51da1d3610094f5c57a626d42cfd7d9c248f53e23420b02b21c695 +-- repository horizon +-- url: https://packages.horizon-haskell.net/ +-- secure: True +-- root-keys: +-- 272e995c7a74de109518100e1422193fe5e5971e52c92b98147c9355b47d7bb6 +-- ea5c1bc0944dabe64d9d68c6813a8141d747cda042b870576d7af63a2326c31b +-- eb47482ddf51da1d3610094f5c57a626d42cfd7d9c248f53e23420b02b21c695 diff --git a/changelog.d/811 b/changelog.d/811 new file mode 100644 index 00000000..4424f39a --- /dev/null +++ b/changelog.d/811 @@ -0,0 +1,2 @@ +synopsis: Add prometheus counter for package imports +prs: #811 diff --git a/environment.sh b/environment.sh index a0935786..082c6794 100755 --- a/environment.sh +++ b/environment.sh @@ -20,7 +20,7 @@ export FLORA_HTTP_PORT=8083 export FLORA_ENVIRONMENT="development" export FLORA_DOMAIN="localhost" -# Either "stdout" or "json" +# Either "stdout", "json" or "json-file" export FLORA_LOGGING_DESTINATION="stdout" # Compatibility mode for Hackage. diff --git a/flora.cabal b/flora.cabal index 498e224c..6b97f1f5 100644 --- a/flora.cabal +++ b/flora.cabal @@ -97,6 +97,7 @@ library Effectful.Poolboy Flora.Environment Flora.Environment.Config + Flora.Environment.Env Flora.Import.Categories Flora.Import.Categories.Tuning Flora.Import.Package @@ -146,6 +147,7 @@ library Flora.Model.User Flora.Model.User.Query Flora.Model.User.Update + Flora.Monitoring Flora.QRCode Flora.Tracing JSON @@ -203,6 +205,7 @@ library , poolboy , postgresql-simple , pretty + , prometheus-client , qrcode-core , qrcode-juicypixels , resource-pool diff --git a/src/core/Flora/Environment.hs b/src/core/Flora/Environment.hs index 3bdda8ed..438e34ec 100644 --- a/src/core/Flora/Environment.hs +++ b/src/core/Flora/Environment.hs @@ -1,58 +1,23 @@ -{-# LANGUAGE PartialTypeSignatures #-} - module Flora.Environment - ( FloraEnv (..) - , DeploymentEnv (..) - , MLTP (..) - , FeatureEnv (..) - , BlobStoreImpl (..) - , TestEnv (..) - , getFloraEnv + ( getFloraEnv , getFloraTestEnv - ) -where + ) where import Colourista.IO (blueMessage) -import Data.Aeson (ToJSON) import Data.ByteString (ByteString) import Data.Pool (Pool) import Data.Pool qualified as Pool import Data.Pool.Introspection (defaultPoolConfig) -import Data.Text import Data.Text.Encoding qualified as Text import Data.Time (NominalDiffTime) -import Data.Word (Word16) import Database.PostgreSQL.Simple qualified as PG import Effectful import Effectful.Fail (Fail) -import Env - ( parse - ) -import Flora.Environment.Config -import GHC.Generics +import Env (parse) --- | The datatype that is used in the application -data FloraEnv = FloraEnv - { pool :: Pool PG.Connection - , dbConfig :: PoolConfig - , jobsPool :: Pool PG.Connection - , httpPort :: Word16 - , domain :: Text - , mltp :: MLTP - , environment :: DeploymentEnv - , features :: FeatureEnv - , config :: FloraConfig - , assets :: Assets - } - deriving stock (Generic) - -data TestEnv = TestEnv - { pool :: Pool PG.Connection - , dbConfig :: PoolConfig - , httpPort :: Word16 - , mltp :: MLTP - } - deriving stock (Generic) +import Flora.Environment.Config +import Flora.Environment.Env +import Flora.Monitoring mkPool :: IOE :> es @@ -69,16 +34,6 @@ mkPool connectionInfo timeout' connections = (realToFrac timeout') connections -data BlobStoreImpl = BlobStoreFS FilePath | BlobStorePure - deriving stock (Generic, Show) - -instance ToJSON BlobStoreImpl - -newtype FeatureEnv = FeatureEnv {blobStoreImpl :: Maybe BlobStoreImpl} - deriving stock (Generic, Show) - -instance ToJSON FeatureEnv - -- In future we'll want to error for conflicting o ptions featureConfigToEnv :: FeatureConfig -> Eff es FeatureEnv featureConfigToEnv FeatureConfig{..} = @@ -94,8 +49,8 @@ configToEnv floraConfig = do pool <- mkPool floraConfig.connectionInfo connectionTimeout connections jobsPool <- mkPool floraConfig.connectionInfo connectionTimeout connections assets <- getAssets floraConfig.environment - liftIO $ print assets featureEnv <- featureConfigToEnv floraConfig.features + metrics <- registerMetrics pure FloraEnv { pool = pool @@ -108,12 +63,14 @@ configToEnv floraConfig = do , features = featureEnv , assets = assets , config = floraConfig + , metrics = metrics } testConfigToTestEnv :: TestConfig -> Eff '[IOE] TestEnv testConfigToTestEnv config@TestConfig{..} = do let PoolConfig{..} = config.dbConfig pool <- mkPool connectionInfo connectionTimeout connections + metrics <- registerMetrics pure TestEnv{..} getFloraEnv :: Eff '[Fail, IOE] FloraEnv diff --git a/src/core/Flora/Environment/Env.hs b/src/core/Flora/Environment/Env.hs new file mode 100644 index 00000000..104bea25 --- /dev/null +++ b/src/core/Flora/Environment/Env.hs @@ -0,0 +1,57 @@ +module Flora.Environment.Env + ( FloraEnv (..) + , Metrics (..) + , DeploymentEnv (..) + , MLTP (..) + , FeatureEnv (..) + , BlobStoreImpl (..) + , TestEnv (..) + ) where + +import Data.Aeson +import Data.Pool (Pool) +import Data.Text (Text) +import Data.Word +import Database.PostgreSQL.Simple qualified as PG +import Flora.Environment.Config +import GHC.Generics +import Prometheus qualified as P + +-- | The datatype that is used in the application +data FloraEnv = FloraEnv + { pool :: Pool PG.Connection + , dbConfig :: PoolConfig + , jobsPool :: Pool PG.Connection + , httpPort :: Word16 + , domain :: Text + , mltp :: MLTP + , environment :: DeploymentEnv + , features :: FeatureEnv + , config :: FloraConfig + , assets :: Assets + , metrics :: Metrics + } + deriving stock (Generic) + +data Metrics = Metrics + { packageImportCounter :: P.Vector P.Label1 P.Counter + } + +data TestEnv = TestEnv + { pool :: Pool PG.Connection + , dbConfig :: PoolConfig + , httpPort :: Word16 + , mltp :: MLTP + , metrics :: Metrics + } + deriving stock (Generic) + +data BlobStoreImpl = BlobStoreFS FilePath | BlobStorePure + deriving stock (Generic, Show) + +instance ToJSON BlobStoreImpl + +newtype FeatureEnv = FeatureEnv {blobStoreImpl :: Maybe BlobStoreImpl} + deriving stock (Generic, Show) + +instance ToJSON FeatureEnv diff --git a/src/core/Flora/Import/Package/Bulk.hs b/src/core/Flora/Import/Package/Bulk.hs index a61f1f91..ebcd9a67 100644 --- a/src/core/Flora/Import/Package/Bulk.hs +++ b/src/core/Flora/Import/Package/Bulk.hs @@ -25,6 +25,7 @@ import Data.Text (Text) import Data.Text qualified as Text import Data.Time (UTCTime) import Data.Time.Clock.POSIX (posixSecondsToUTCTime) +import Distribution.Types.Version (Version) import Effectful import Effectful.FileSystem (FileSystem) import Effectful.FileSystem qualified as FileSystem @@ -33,9 +34,11 @@ import Effectful.Log (Log) import Effectful.Log qualified as Log import Effectful.Poolboy import Effectful.PostgreSQL.Transact.Effect (DB) +import Effectful.Reader.Static (Reader) import Effectful.State.Static.Shared (State) import Effectful.Time (Time) import GHC.Conc (numCapabilities) +import GHC.Records import Streamly.Data.Fold qualified as SFold import Streamly.Data.Stream (Stream) import Streamly.Data.Stream.Prelude (maxThreads, ordered) @@ -45,7 +48,7 @@ import System.Directory qualified as System import System.FilePath import UnliftIO.Exception (finally) -import Distribution.Types.Version (Version) +import Flora.Environment.Env import Flora.Import.Package ( extractPackageDataFromCabal , loadContent @@ -60,6 +63,7 @@ import Flora.Model.PackageIndex.Update qualified as Update import Flora.Model.Release.Query qualified as Query import Flora.Model.Release.Update qualified as Update import Flora.Model.User +import Flora.Monitoring -- | Same as 'importAllFilesInDirectory' but accepts a relative path to the current working directory importAllFilesInRelativeDirectory @@ -70,6 +74,9 @@ importAllFilesInRelativeDirectory , IOE :> es , Poolboy :> es , State (Set (Namespace, PackageName, Version)) :> es + , Reader r :> es + , HasField "metrics" r Metrics + , HasField "mltp" r MLTP ) => UserId -> (Text, Text) @@ -86,6 +93,9 @@ importFromIndex , DB :> es , IOE :> es , State (Set (Namespace, PackageName, Version)) :> es + , Reader r :> es + , HasField "metrics" r Metrics + , HasField "mltp" r MLTP ) => UserId -> Text @@ -142,6 +152,9 @@ importAllFilesInDirectory , IOE :> es , Poolboy :> es , State (Set (Namespace, PackageName, Version)) :> es + , Reader r :> es + , HasField "metrics" r Metrics + , HasField "mltp" r MLTP ) => UserId -> (Text, Text) @@ -154,13 +167,16 @@ importAllFilesInDirectory user (repositoryName, _repositoryURL) dir = do importFromStream user (repositoryName, packages) (findAllCabalFilesInDirectory dir) importFromStream - :: forall es + :: forall es r . ( Time :> es , Log :> es , DB :> es , IOE :> es , Poolboy :> es , State (Set (Namespace, PackageName, Version)) :> es + , Reader r :> es + , HasField "metrics" r Metrics + , HasField "mltp" r MLTP ) => UserId -> (Text, Set PackageName) @@ -182,16 +198,18 @@ importFromStream userId repository@(repositoryName, _) stream = do Update.updatePackageIndexByName repositoryName timestamp ) displayStats processedPackageCount + increasePackageImportCounterBy + (fromIntegral @Int @Double processedPackageCount) + repositoryName where displayCount :: SFold.Fold (Eff es) a Int displayCount = flip SFold.foldlM' (pure 0) $ - \previousCount _ -> + \previousCount _ -> do let currentCount = previousCount + 1 - in do - when (currentCount `mod` 400 == 0) $ - displayStats currentCount - pure currentCount + batchAmount = 400 + when (currentCount `mod` batchAmount == 0) $ displayStats currentCount + pure currentCount displayStats :: IOE :> es diff --git a/src/core/Flora/Model/Job.hs b/src/core/Flora/Model/Job.hs index 969d6693..7c5a279b 100644 --- a/src/core/Flora/Model/Job.hs +++ b/src/core/Flora/Model/Job.hs @@ -86,7 +86,7 @@ data FloraOddJobs | FetchPackageDeprecationList | FetchReleaseDeprecationList PackageName (Vector ReleaseId) | RefreshLatestVersions - | RefreshIndexes + | RefreshIndex Text deriving stock (Generic) -- TODO: Upstream these two ToJSON instances @@ -106,4 +106,5 @@ instance ToJSON LogEvent where LogWebUIRequest -> toJSON ("web-ui-request" :: Text) LogKillJobSuccess job -> toJSON ("kill-success" :: Text, job) LogKillJobFailed job -> toJSON ("kill-failed" :: Text, job) + LogDeletionPoll data_ -> toJSON ("log-deletion-poll" :: Text, data_) LogText other -> toJSON ("other" :: Text, other) diff --git a/src/core/Flora/Model/PackageIndex/Query.hs b/src/core/Flora/Model/PackageIndex/Query.hs index e8d34fd4..b9c1f30e 100644 --- a/src/core/Flora/Model/PackageIndex/Query.hs +++ b/src/core/Flora/Model/PackageIndex/Query.hs @@ -4,7 +4,9 @@ module Flora.Model.PackageIndex.Query where import Data.Text (Text) -import Database.PostgreSQL.Entity (selectOneByField) +import Data.Vector (Vector) +import Data.Vector qualified as Vector +import Database.PostgreSQL.Entity import Database.PostgreSQL.Entity.Types import Database.PostgreSQL.Simple (Only (..)) import Effectful @@ -19,3 +21,9 @@ getPackageIndexByName repository = r -> r in dbtToEff $ selectOneByField [field| repository |] (Only index) + +listPackageIndexes :: DB :> es => Eff es (Vector PackageIndex) +listPackageIndexes = + dbtToEff $ + selectOrderBy @PackageIndex $ + Vector.fromList [([field| repository |], ASC)] diff --git a/src/core/Flora/Monitoring.hs b/src/core/Flora/Monitoring.hs new file mode 100644 index 00000000..d2285d14 --- /dev/null +++ b/src/core/Flora/Monitoring.hs @@ -0,0 +1,88 @@ +module Flora.Monitoring + ( increaseCounter + , increasePackageImportCounter + , registerMetrics + , increaseCounterBy + , increasePackageImportCounterBy + ) where + +import Control.Monad (void, when) +import Data.Text +import Effectful +import Effectful.Reader.Static (Reader, ask) +import GHC.Records +import Prometheus +import Prometheus qualified as P + +import Flora.Environment.Env + +registerMetrics :: IOE :> es => Eff es Metrics +registerMetrics = do + let packageImportCount = + P.vector "package_index" $ + P.counter + P.Info + { metricName = "flora_packages_imported_total" + , metricHelp = "Packages imported and their index" + } + packageImportCounter <- P.register packageImportCount + pure Metrics{..} + +increaseCounter + :: forall r es label + . ( HasField "mltp" r MLTP + , Reader r :> es + , Label label + , IOE :> es + ) + => Vector label Counter + -> label + -> Eff es () +increaseCounter promVector label = do + env <- ask + let mltpConf = env.mltp + when mltpConf.prometheusEnabled $ + liftIO $ + withLabel promVector label incCounter + +increaseCounterBy + :: ( HasField "mltp" r MLTP + , Reader r :> es + , Label label + , IOE :> es + ) + => Double + -> Vector label Counter + -> label + -> Eff es () +increaseCounterBy value promVector label = do + env <- ask + let mltpConf = env.mltp + when mltpConf.prometheusEnabled $ + liftIO $ + withLabel promVector label (\c -> void $ addCounter c value) + +increasePackageImportCounter + :: ( HasField "metrics" r Metrics + , HasField "mltp" r MLTP + , Reader r :> es + , IOE :> es + ) + => Text + -> Eff es () +increasePackageImportCounter repository = do + env <- ask + increaseCounter env.metrics.packageImportCounter repository + +increasePackageImportCounterBy + :: ( HasField "metrics" r Metrics + , HasField "mltp" r MLTP + , Reader r :> es + , IOE :> es + ) + => Double + -> Text + -> Eff es () +increasePackageImportCounterBy value repository = do + env <- ask + increaseCounterBy value env.metrics.packageImportCounter repository diff --git a/src/jobs-worker/FloraJobs/Runner.hs b/src/jobs-worker/FloraJobs/Runner.hs index 9c039e0e..81decad6 100644 --- a/src/jobs-worker/FloraJobs/Runner.hs +++ b/src/jobs-worker/FloraJobs/Runner.hs @@ -6,11 +6,14 @@ import Control.Monad.IO.Class import Data.Aeson (Result (..), fromJSON, toJSON) import Data.Function import Data.Maybe (fromJust) +import Data.Set (Set) import Data.Set qualified as Set import Data.Text qualified as Text import Data.Text.Display +import Data.Text.HTML qualified as HTML import Data.Vector (Vector) import Data.Vector qualified as Vector +import Distribution.Types.Version (Version) import Effectful (Eff, IOE, type (:>)) import Effectful.FileSystem (FileSystem) import Effectful.FileSystem qualified as FileSystem @@ -19,6 +22,7 @@ import Effectful.Poolboy (Poolboy) import Effectful.PostgreSQL.Transact.Effect (DB) import Effectful.Process.Typed import Effectful.Reader.Static (Reader) +import Effectful.State.Static.Shared (State) import Effectful.Time (Time) import Log import Network.HTTP.Types (gone410, notFound404, statusCode) @@ -26,10 +30,8 @@ import OddJobs.Job (Job (..)) import Servant.Client (ClientError (..)) import Servant.Client.Core (ResponseF (..)) -import Data.Set (Set) -import Data.Text.HTML qualified as HTML -import Distribution.Types.Version (Version) -import Effectful.State.Static.Shared (State) +import Data.Text +import Flora.Environment.Env import Flora.Import.Package (coreLibraries, persistImportOutput) import Flora.Import.Package.Bulk qualified as Import import Flora.Model.BlobIndex.Update qualified as Update @@ -61,14 +63,11 @@ runner job = localDomain "job-runner" $ FetchTarball x -> fetchTarball x FetchUploadTime x -> fetchUploadTime x FetchChangelog x -> fetchChangeLog x - ImportPackage x -> - persistImportOutput x + ImportPackage x -> persistImportOutput x FetchPackageDeprecationList -> fetchPackageDeprecationList - FetchReleaseDeprecationList packageName releases -> - fetchReleaseDeprecationList packageName releases - RefreshLatestVersions -> - Update.refreshLatestVersions - RefreshIndexes -> refreshIndexes + FetchReleaseDeprecationList packageName releases -> fetchReleaseDeprecationList packageName releases + RefreshLatestVersions -> Update.refreshLatestVersions + RefreshIndex indexName -> refreshIndex indexName fetchChangeLog :: ChangelogJobPayload -> JobsRunner () fetchChangeLog payload@ChangelogJobPayload{packageName, packageVersion, releaseId} = @@ -222,33 +221,33 @@ assignNamespace = else PackageAlternative (Namespace "hackage") p ) -refreshIndexes +refreshIndex :: ( Time :> es , DB :> es - , TypedProcess :> es , Log :> es , Poolboy :> es + , TypedProcess :> es , IOE :> es , FileSystem :> es , State (Set (Namespace, PackageName, Version)) :> es + , Reader FloraEnv :> es ) - => Eff es () -refreshIndexes = do + => Text + -> Eff es () +refreshIndex indexName = do + let repoPath = + if indexName == "hackage" + then "hackage.haskell.org" + else Text.unpack indexName runProcess_ $ shell "cabal update --project-file cabal.project.repositories" - let packageIndexes = - [ ("hackage", "hackage.haskell.org") - , ("cardano", "cardano") - , ("horizon", "horizon") - ] user <- fromJust <$> Query.getUserByUsername "hackage-user" - forM_ packageIndexes $ \(indexName, repoPath) -> do - homeDir <- FileSystem.getHomeDirectory - let path = homeDir <> "/.cabal/packages/" <> repoPath <> "/01-index.tar.gz" - mPackageIndex <- Query.getPackageIndexByName indexName - case mPackageIndex of - Nothing -> do - Log.logAttention "Package index not found" $ - object ["package_index" .= indexName] - error $ Text.unpack $ "Package index " <> indexName <> " not found in the database!" - Just _ -> - Import.importFromIndex user.userId indexName path + homeDir <- FileSystem.getHomeDirectory + let path = homeDir <> "/.cabal/packages/" <> repoPath <> "/01-index.tar.gz" + mPackageIndex <- Query.getPackageIndexByName indexName + case mPackageIndex of + Nothing -> do + Log.logAttention "Package index not found" $ + object ["package_index" .= indexName] + error $ Text.unpack $ "Package index " <> indexName <> " not found in the database!" + Just _ -> + Import.importFromIndex user.userId indexName path diff --git a/src/jobs-worker/FloraJobs/Scheduler.hs b/src/jobs-worker/FloraJobs/Scheduler.hs index e88d218a..435635f7 100644 --- a/src/jobs-worker/FloraJobs/Scheduler.hs +++ b/src/jobs-worker/FloraJobs/Scheduler.hs @@ -9,7 +9,7 @@ module FloraJobs.Scheduler , schedulePackageDeprecationListJob , scheduleReleaseDeprecationListJob , scheduleRefreshLatestVersions - , scheduleRefreshIndexes + , scheduleRefreshIndex , checkIfIndexRefreshJobIsPlanned , jobTableName -- prefer using smart constructors. @@ -19,12 +19,14 @@ module FloraJobs.Scheduler ) where +import Control.Monad import Data.Aeson (ToJSON) import Data.Pool +import Data.Text (Text) import Data.Time qualified as Time import Data.Vector (Vector) +import Data.Vector qualified as Vector import Database.PostgreSQL.Entity.DBT -import Database.PostgreSQL.Simple (Only (..)) import Database.PostgreSQL.Simple qualified as PG import Database.PostgreSQL.Simple.SqlQQ (sql) import Distribution.Types.Version @@ -34,8 +36,11 @@ import Effectful.PostgreSQL.Transact.Effect import Log import OddJobs.Job (Job (..), createJob, scheduleJob) +import Database.PostgreSQL.Simple.Types import Flora.Model.Job import Flora.Model.Package +import Flora.Model.PackageIndex.Query qualified as Query +import Flora.Model.PackageIndex.Types import Flora.Model.Release.Types import FloraJobs.Types @@ -76,10 +81,10 @@ scheduleReleaseDeprecationListJob pool (package, releaseIds) = scheduleRefreshLatestVersions :: Pool PG.Connection -> IO Job scheduleRefreshLatestVersions pool = createJobWithResource pool RefreshLatestVersions -scheduleRefreshIndexes :: Pool PG.Connection -> IO Job -scheduleRefreshIndexes pool = withResource pool $ \conn -> do +scheduleRefreshIndex :: Pool PG.Connection -> Text -> IO Job +scheduleRefreshIndex pool indexName = withResource pool $ \conn -> do now <- Time.getCurrentTime - scheduleJob conn jobTableName RefreshIndexes (Time.addUTCTime Time.nominalDay now) + scheduleJob conn jobTableName (RefreshIndex indexName) (Time.addUTCTime Time.nominalDay now) createJobWithResource :: ToJSON p => Pool PG.Connection -> p -> IO Job createJobWithResource pool job = @@ -88,24 +93,25 @@ createJobWithResource pool job = checkIfIndexRefreshJobIsPlanned :: ( DB :> es , Log :> es + , IOE :> es ) - => Eff es Bool -checkIfIndexRefreshJobIsPlanned = do + => Pool PG.Connection + -> Eff es () +checkIfIndexRefreshJobIsPlanned pool = do Log.logInfo_ "Checking if the index refresh job is planned…" - (result :: Maybe (Only Int)) <- + indexes <- Query.listPackageIndexes + (result' :: Vector (Only Text)) <- dbtToEff $ - queryOne_ + query_ Select [sql| - select count(*) - from "oddjobs" - where payload ->> 'tag' = 'RefreshIndexes' - and status = 'queued' - |] - case result of - Just (Only 1) -> do - Log.logInfo_ "Index refresh job is planned" - pure True - _ -> do - Log.logInfo_ "Index refresh job not not planned" - pure False + select payload ->> 'contents' + from "oddjobs" + where payload ->> 'tag' = 'RefreshIndex' + and status = 'queued' + |] + let result = fmap fromOnly result' + forM_ indexes $ \index -> + when (Vector.notElem index.repository result) $ do + Log.logInfo "Scheduling index refresh" $ object ["index" .= index.repository] + void $ liftIO $ scheduleRefreshIndex pool index.repository diff --git a/src/jobs-worker/FloraJobs/Types.hs b/src/jobs-worker/FloraJobs/Types.hs index d4b97efe..6af52599 100644 --- a/src/jobs-worker/FloraJobs/Types.hs +++ b/src/jobs-worker/FloraJobs/Types.hs @@ -15,13 +15,15 @@ import Database.PostgreSQL.Simple qualified as PG import Database.PostgreSQL.Simple.Types (QualifiedIdentifier) import Distribution.Types.Version (Version) import Effectful +import Effectful.Concurrent.Async import Effectful.FileSystem import Effectful.Log hiding (LogLevel) import Effectful.Log qualified as LogEff hiding (LogLevel) import Effectful.Poolboy import Effectful.PostgreSQL.Transact.Effect (DB, runDB) import Effectful.Process.Typed -import Effectful.Reader.Static (Reader, runReader) +import Effectful.Reader.Static (Reader) +import Effectful.Reader.Static qualified as Reader import Effectful.State.Static.Shared (State) import Effectful.State.Static.Shared qualified as State import Effectful.Time (Time, runTime) @@ -34,8 +36,8 @@ import OddJobs.Job (Config (..), Job, LogEvent (..), LogLevel (..)) import OddJobs.Types (ConcurrencyControl (..), UIConfig (..)) import Data.Set (Set) -import Flora.Environment import Flora.Environment.Config +import Flora.Environment.Env import Flora.Logging qualified as Logging import Flora.Model.BlobStore.API import Flora.Model.Job () @@ -52,16 +54,24 @@ type JobsRunner = , TypedProcess , FileSystem , State (Set (Namespace, PackageName, Version)) + , Reader FloraEnv + , Concurrent , IOE ] -runJobRunner :: Pool Connection -> JobsRunnerEnv -> FloraEnv -> Logger -> JobsRunner a -> IO a +runJobRunner + :: Pool Connection + -> JobsRunnerEnv + -> FloraEnv + -> Logger + -> JobsRunner a + -> IO a runJobRunner pool runnerEnv floraEnv logger jobRunner = jobRunner & withUnliftStrategy (ConcUnlift Ephemeral Unlimited) & runDB pool & runPoolboy (poolboySettingsWith floraEnv.dbConfig.connections) - & runReader runnerEnv + & Reader.runReader runnerEnv & ( case floraEnv.features.blobStoreImpl of Just (BlobStoreFS fp) -> runBlobStoreFS fp _ -> runBlobStorePure @@ -71,6 +81,8 @@ runJobRunner pool runnerEnv floraEnv logger jobRunner = & runTypedProcess & runFileSystem & State.evalState mempty + & Reader.runReader floraEnv + & runConcurrent & runEff data OddJobException where diff --git a/src/web/FloraWeb/Common/Auth.hs b/src/web/FloraWeb/Common/Auth.hs index 329ff7a1..40804e7e 100644 --- a/src/web/FloraWeb/Common/Auth.hs +++ b/src/web/FloraWeb/Common/Auth.hs @@ -8,11 +8,16 @@ module FloraWeb.Common.Auth ) where +import Control.Monad.Except qualified as T import Data.Function ((&)) +import Data.Kind (Type) import Data.List qualified as List import Data.Text (Text) +import Data.Text.Encoding qualified as Text import Data.UUID qualified as UUID +import Data.UUID.V4 qualified as UUID import Effectful +import Effectful.Dispatch.Static import Effectful.Error.Static (Error, runErrorNoCallStack, throwError) import Effectful.PostgreSQL.Transact.Effect (DB) import Effectful.PostgreSQL.Transact.Effect qualified as DB @@ -24,12 +29,7 @@ import Servant.Server import Servant.Server.Experimental.Auth (AuthHandler, mkAuthHandler) import Web.Cookie -import Control.Monad.Except qualified as T -import Data.Kind (Type) -import Data.Text.Encoding qualified as Text -import Data.UUID.V4 qualified as UUID -import Effectful.Dispatch.Static -import Flora.Environment +import Flora.Environment.Env import Flora.Logging qualified as Logging import Flora.Model.PersistentSession import Flora.Model.User diff --git a/src/web/FloraWeb/Common/Tracing.hs b/src/web/FloraWeb/Common/Tracing.hs index ba11fd8d..22aeecb5 100644 --- a/src/web/FloraWeb/Common/Tracing.hs +++ b/src/web/FloraWeb/Common/Tracing.hs @@ -16,7 +16,7 @@ import System.Log.Raven (initRaven, register, silentFallback) import System.Log.Raven.Transport.HttpConduit (sendRecord) import System.Log.Raven.Types (SentryLevel (Error), SentryRecord (..)) -import Flora.Environment +import Flora.Environment.Env onException :: Logger -> DeploymentEnv -> MLTP -> Maybe Request -> SomeException -> IO () onException logger environment mltp mRequest exception = diff --git a/src/web/FloraWeb/LiveReload.hs b/src/web/FloraWeb/LiveReload.hs index b9dcd10a..6dd7ae0a 100644 --- a/src/web/FloraWeb/LiveReload.hs +++ b/src/web/FloraWeb/LiveReload.hs @@ -1,12 +1,12 @@ module FloraWeb.LiveReload where import Data.IORef +import Data.Text (Text) import Effectful +import Servant.API (Header, Headers, NoContent (..)) -import Data.Text (Text) -import Flora.Environment (DeploymentEnv (..)) +import Flora.Environment.Env (DeploymentEnv (..)) import FloraWeb.Common.Utils -import Servant.API (Header, Headers, NoContent (..)) livereloadHandler :: IOE :> es diff --git a/src/web/FloraWeb/Pages/Server/Admin.hs b/src/web/FloraWeb/Pages/Server/Admin.hs index f5baacdf..0d8e6ecb 100644 --- a/src/web/FloraWeb/Pages/Server/Admin.hs +++ b/src/web/FloraWeb/Pages/Server/Admin.hs @@ -16,7 +16,7 @@ import OddJobs.Types qualified as OddJobs import Optics.Core import Servant (HasServer (..), Headers (..)) -import Flora.Environment (FeatureEnv (..), FloraEnv (..)) +import Flora.Environment.Env (FeatureEnv (..), FloraEnv (..)) import Flora.Model.Admin.Report import Flora.Model.Release.Query qualified as Query import Flora.Model.User @@ -24,7 +24,13 @@ import FloraJobs.Scheduler import FloraWeb.Common.Auth import FloraWeb.Common.Utils (handlerToEff, redirect) import FloraWeb.Pages.Routes.Admin -import FloraWeb.Pages.Templates (ActiveElements (..), TemplateEnv (..), defaultTemplateEnv, render, templateFromSession) +import FloraWeb.Pages.Templates + ( ActiveElements (..) + , TemplateEnv (..) + , defaultTemplateEnv + , render + , templateFromSession + ) import FloraWeb.Pages.Templates.Admin qualified as Templates import FloraWeb.Types (FloraEff, fetchFloraEnv) diff --git a/src/web/FloraWeb/Pages/Server/Categories.hs b/src/web/FloraWeb/Pages/Server/Categories.hs index adb92f92..f291fffe 100644 --- a/src/web/FloraWeb/Pages/Server/Categories.hs +++ b/src/web/FloraWeb/Pages/Server/Categories.hs @@ -2,14 +2,14 @@ module FloraWeb.Pages.Server.Categories where import Data.Text (Text) import Effectful (Eff, IOE, (:>)) +import Effectful.Error.Static (Error) +import Effectful.PostgreSQL.Transact.Effect (DB) +import Effectful.Reader.Static (Reader) import Lucid (Html) import Network.HTTP.Types (notFound404) import Servant (Headers (..), ServerError, ServerT) -import Effectful.Error.Static (Error) -import Effectful.PostgreSQL.Transact.Effect (DB) -import Effectful.Reader.Static (Reader) -import Flora.Environment (FeatureEnv) +import Flora.Environment.Env (FeatureEnv) import Flora.Model.Category.Query qualified as Query import Flora.Model.Category.Types (Category (..)) import Flora.Model.Package.Query qualified as Query diff --git a/src/web/FloraWeb/Pages/Server/Packages.hs b/src/web/FloraWeb/Pages/Server/Packages.hs index 43e93db9..203f2aa4 100644 --- a/src/web/FloraWeb/Pages/Server/Packages.hs +++ b/src/web/FloraWeb/Pages/Server/Packages.hs @@ -33,7 +33,7 @@ import Servant.Server (err404) import Advisories.Model.Affected.Query qualified as Query import Advisories.Model.Affected.Types -import Flora.Environment (FeatureEnv (..)) +import Flora.Environment.Env (FeatureEnv (..)) import Flora.Model.BlobIndex.Query qualified as Query import Flora.Model.BlobStore.API (BlobStoreAPI) import Flora.Model.Package diff --git a/src/web/FloraWeb/Pages/Server/Settings.hs b/src/web/FloraWeb/Pages/Server/Settings.hs index 7f1ebcc3..f880de62 100644 --- a/src/web/FloraWeb/Pages/Server/Settings.hs +++ b/src/web/FloraWeb/Pages/Server/Settings.hs @@ -7,17 +7,17 @@ import Control.Monad.IO.Class import Data.ByteString.Base32 qualified as Base32 import Data.Text.Encoding qualified as Text import Effectful (Eff, IOE, (:>)) +import Effectful.Log (Log) +import Effectful.PostgreSQL.Transact.Effect (DB) import Effectful.Reader.Static (Reader) +import Effectful.Time (Time) import Log qualified import Lucid import Optics.Core import Sel.HMAC.SHA256 qualified as HMAC import Servant (HasServer (..), Headers (..), Union, WithStatus (..), respond) -import Effectful.Log (Log) -import Effectful.PostgreSQL.Transact.Effect (DB) -import Effectful.Time (Time) -import Flora.Environment +import Flora.Environment.Env import Flora.Model.User import Flora.Model.User.Update qualified as Update import Flora.QRCode qualified as QRCode diff --git a/src/web/FloraWeb/Pages/Templates.hs b/src/web/FloraWeb/Pages/Templates.hs index 0e75cf1b..d1b1867e 100644 --- a/src/web/FloraWeb/Pages/Templates.hs +++ b/src/web/FloraWeb/Pages/Templates.hs @@ -13,7 +13,7 @@ import Data.ByteString.Lazy import Data.Text.Display import Lucid -import Flora.Environment (DeploymentEnv (..)) +import Flora.Environment.Env (DeploymentEnv (..)) import FloraWeb.Components.Alert qualified as Alert import FloraWeb.Components.Header (header) import FloraWeb.Pages.Templates.Types as Types diff --git a/src/web/FloraWeb/Pages/Templates/Error.hs b/src/web/FloraWeb/Pages/Templates/Error.hs index 47ddd07f..d24ea97f 100644 --- a/src/web/FloraWeb/Pages/Templates/Error.hs +++ b/src/web/FloraWeb/Pages/Templates/Error.hs @@ -13,7 +13,7 @@ import Data.Kind (Type) import Effectful import Effectful.Error.Static (Error, throwError) import Effectful.Reader.Static (Reader) -import Flora.Environment (FeatureEnv) +import Flora.Environment.Env (FeatureEnv) import Flora.Model.User (User) import FloraWeb.Pages.Templates import FloraWeb.Session diff --git a/src/web/FloraWeb/Pages/Templates/Packages.hs b/src/web/FloraWeb/Pages/Templates/Packages.hs index b80036e8..5d4b5f18 100644 --- a/src/web/FloraWeb/Pages/Templates/Packages.hs +++ b/src/web/FloraWeb/Pages/Templates/Packages.hs @@ -56,7 +56,7 @@ import Text.PrettyPrint (Doc, hcat, render) import Text.PrettyPrint qualified as PP import Advisories.Model.Affected.Types -import Flora.Environment (FeatureEnv (..)) +import Flora.Environment.Env (FeatureEnv (..)) import Flora.Model.Category.Types import Flora.Model.Package import Flora.Model.Release.Types diff --git a/src/web/FloraWeb/Pages/Templates/Screens/Home.hs b/src/web/FloraWeb/Pages/Templates/Screens/Home.hs index aac9ad09..44f0e449 100644 --- a/src/web/FloraWeb/Pages/Templates/Screens/Home.hs +++ b/src/web/FloraWeb/Pages/Templates/Screens/Home.hs @@ -8,7 +8,7 @@ import Data.Text (Text) import Lucid import PyF -import Flora.Environment +import Flora.Environment.Env import FloraWeb.Components.MainSearchBar (mainSearchBar) import FloraWeb.Pages.Templates.Types diff --git a/src/web/FloraWeb/Pages/Templates/Types.hs b/src/web/FloraWeb/Pages/Templates/Types.hs index e71d91d2..8158f87b 100644 --- a/src/web/FloraWeb/Pages/Templates/Types.hs +++ b/src/web/FloraWeb/Pages/Templates/Types.hs @@ -15,16 +15,16 @@ where import Control.Monad.Identity import Control.Monad.Reader (ReaderT) import Data.Text (Text) +import Data.Text.Display import Data.UUID qualified as UUID +import Effectful +import Effectful.Reader.Static (Reader, ask) import GHC.Generics import Lucid import Optics.Core -import Data.Text.Display -import Effectful -import Effectful.Reader.Static (Reader, ask) -import Flora.Environment import Flora.Environment.Config (Assets) +import Flora.Environment.Env import Flora.Model.PersistentSession (PersistentSessionId (..)) import Flora.Model.User import FloraWeb.Common.Auth diff --git a/src/web/FloraWeb/Server.hs b/src/web/FloraWeb/Server.hs index 18b2084d..6b9374ea 100644 --- a/src/web/FloraWeb/Server.hs +++ b/src/web/FloraWeb/Server.hs @@ -60,15 +60,14 @@ import Servant import Servant.OpenApi import Servant.Server.Generic (AsServerT) -import Flora.Environment +import Flora.Environment (getFloraEnv) +import Flora.Environment.Config (Assets, DeploymentEnv (..)) +import Flora.Environment.Env ( BlobStoreImpl (..) - , DeploymentEnv , FeatureEnv (..) , FloraEnv (..) , MLTP (..) - , getFloraEnv ) -import Flora.Environment.Config (Assets, DeploymentEnv (..)) import Flora.Logging qualified as Logging import Flora.Model.BlobStore.API import Flora.Tracing qualified as Tracing @@ -115,6 +114,7 @@ runFlora = blueMessage $ "🔥 Exposing Prometheus metrics at " <> baseURL <> "/metrics" void $ P.register P.ghcMetrics void $ P.register P.procMetrics + void $ P.register (P.counter (P.Info "flora_imported_packages_total" "The number of imported packages")) liftIO $ when env.mltp.zipkinEnabled (blueMessage "🖊️ Connecting to Zipkin endpoint") liftIO $ when (env.environment == Development) (blueMessage "🔁 Live reloading enabled") let withLogger = Logging.makeLogger env.mltp.logger @@ -159,6 +159,10 @@ runServer appLogger floraEnv = do unsafeEff_ $ Safe.withException (startJobRunner oddJobsCfg) (logException floraEnv.environment appLogger) loggingMiddleware <- Logging.runLog floraEnv.environment appLogger WaiLog.mkLogMiddleware + let prometheusMiddleware = + if floraEnv.mltp.prometheusEnabled + then WaiMetrics.prometheus WaiMetrics.def + else id oddJobsEnv <- OddJobs.mkEnv oddjobsUiCfg ("/admin/odd-jobs/" <>) let webEnv = WebEnv floraEnv webEnvStore <- liftIO $ newWebEnvStore webEnv @@ -178,8 +182,8 @@ runServer appLogger floraEnv = do $ heartbeatMiddleware . loggingMiddleware . const - $ WaiMetrics.prometheus WaiMetrics.def - $ P.prometheusMiddleware P.defaultMetrics (Proxy @ServerRoutes) server + $ P.prometheusMiddleware P.defaultMetrics (Proxy @ServerRoutes) + $ prometheusMiddleware server mkServer :: Logger diff --git a/src/web/FloraWeb/Session.hs b/src/web/FloraWeb/Session.hs index 14b23c70..a0f53972 100644 --- a/src/web/FloraWeb/Session.hs +++ b/src/web/FloraWeb/Session.hs @@ -16,7 +16,7 @@ import Effectful.Internal.Monad (unsafeEff_) import Servant (Header, Headers, addHeader) import Web.Cookie -import Flora.Environment (FloraEnv) +import Flora.Environment.Env (FloraEnv) import Flora.Model.PersistentSession import FloraWeb.Common.Auth.Types import FloraWeb.Types (fetchFloraEnv) diff --git a/src/web/FloraWeb/Types.hs b/src/web/FloraWeb/Types.hs index 46a60476..d2087bbd 100644 --- a/src/web/FloraWeb/Types.hs +++ b/src/web/FloraWeb/Types.hs @@ -18,6 +18,7 @@ import Control.Monad.IO.Class import Control.Monad.Time (MonadTime (..)) import Data.Text.Encoding qualified as TE import Effectful +import Effectful.Concurrent (Concurrent) import Effectful.Error.Static (Error) import Effectful.Log (Log) import Effectful.PostgreSQL.Transact.Effect (DB) @@ -29,8 +30,7 @@ import GHC.Generics import Servant (FromHttpApiData (..), Handler, ServerError) import Web.Cookie -import Effectful.Concurrent (Concurrent) -import Flora.Environment +import Flora.Environment.Env import Flora.Model.BlobStore.API newtype WebEnvStore = WebEnvStore (MVar WebEnv) diff --git a/test/Flora/ImportSpec.hs b/test/Flora/ImportSpec.hs index 8db6e5e3..82fb5cd3 100644 --- a/test/Flora/ImportSpec.hs +++ b/test/Flora/ImportSpec.hs @@ -7,6 +7,7 @@ import Data.Text (Text) import Log.Backend.StandardOutput (withStdOutLogger) import Optics.Core +import Flora.Environment.Env import Flora.Import.Package (chooseNamespace) import Flora.Import.Package.Bulk import Flora.Model.Package.Query qualified as Query diff --git a/test/Flora/TestUtils.hs b/test/Flora/TestUtils.hs index 529d0883..fe79da50 100644 --- a/test/Flora/TestUtils.hs +++ b/test/Flora/TestUtils.hs @@ -135,8 +135,8 @@ import Test.Tasty.HUnit qualified as Test import Effectful.State.Static.Shared (State) import Effectful.State.Static.Shared qualified as State -import Flora.Environment import Flora.Environment.Config +import Flora.Environment.Env import Flora.Import.Package.Bulk (importAllFilesInRelativeDirectory) import Flora.Logging qualified as Logging import Flora.Model.BlobStore.API @@ -174,7 +174,20 @@ import Flora.Model.User import Flora.Model.User.Query qualified as Query import Flora.Model.User.Update qualified as Update -type TestEff = Eff '[Trace, FileSystem, Poolboy, Fail, BlobStoreAPI, Reader TestEnv, DB, Log, Time, State (Set (Namespace, PackageName, Version)), IOE] +type TestEff = + Eff + '[ Trace + , FileSystem + , Poolboy + , Fail + , BlobStoreAPI + , Reader TestEnv + , DB + , Log + , Time + , State (Set (Namespace, PackageName, Version)) + , IOE + ] data Fixtures = Fixtures { hackageUser :: User