Odd jobs tweaks (#140)

* Refactor the jobs code * Don't throw an exception when there is no README * style and freeze * Beautiful CSS for package READMEs
flora-pm · Jun 26, 2022 · 1a2e963 · 1a2e963
1 parent 5f7b936
commit 1a2e963
Show file tree

Hide file tree

Showing 15 changed files with 421 additions and 193 deletions.
diff --git a/assets/css/app.css b/assets/css/app.css
@@ -2,6 +2,8 @@
 @import "tailwindcss/components";
 @import "tailwindcss/utilities";
 
+@import "package-readme.css";
+
 @layer components {
   .larger-container {
     @apply max-w-7xl mx-auto px-2;
@@ -152,7 +154,8 @@ div[class="bullets"] {
   @apply bg-gray-7 dark:bg-blue-1;
 }
 .package-readme-column{
-    margin: 0 2em;
+  margin: 0 2em;
+  overflow: auto;
 }
 
 ul.package-list a {

diff --git a/assets/css/package-readme.css b/assets/css/package-readme.css
@@ -0,0 +1,59 @@
+.package-readme{
+  overflow-wrap: break-word;
+  box-sizing: border-box;
+
+  h1 {
+    border-bottom: 1px solid #354561;
+    padding-bottom: 0.3em;
+    margin-bottom: 16px;
+    font-size: 2em;
+    a {
+      display: inline-block;
+    }
+  }
+
+  img {
+    max-width: 100%;
+    box-sizing: content-box;
+  }
+
+  h2 {
+    border-bottom: 1px solid #354561;
+    margin-bottom: 16px;
+    padding-bottom: 0.3em;
+    font-size: 1.5em;
+  }
+
+  ul {
+    margin-bottom: 16px;
+
+    li {
+      overflow-wrap: break-word;
+      list-style-type: disc;
+      list-style-position: inside;
+    }
+    li + li {
+      margin-top: .25em;
+    }
+  }
+
+  p {
+    overflow-wrap: break-word;
+    margin-bottom: 16px;
+    box-sizing: border-box;
+    a {
+      display: inline-block;
+    }
+  }
+
+  pre {
+    line-height: 1.45;
+    padding: 16px;
+    font-family: monospace;
+    border-radius: 6px;
+    overflow: auto;
+    font-size: 90%;
+    background-color: #354561;
+    margin-bottom: 16px;
+  }
+}
diff --git a/cabal.project.freeze b/cabal.project.freeze
@@ -8,6 +8,7 @@ constraints: any.Cabal ==3.6.3.0,
              PyF -python_test,
              any.QuickCheck ==2.14.2,
              QuickCheck -old-random +templatehaskell,
+             any.RSA ==2.4.1,
              any.SHA ==1.6.4.4,
              SHA -exe,
              any.StateVar ==1.2.2,
@@ -34,12 +35,14 @@ constraints: any.Cabal ==3.6.3.0,
              attoparsec -developer,
              any.attoparsec-iso8601 ==1.0.2.1,
              attoparsec-iso8601 -developer -fast,
+             any.authenticate-oauth ==1.7,
              any.auto-update ==0.1.6,
              any.barbies ==2.0.3.1,
              any.base ==4.14.3.0,
              any.base-compat ==0.12.1,
              any.base-compat-batteries ==0.12.1,
              any.base-orphans ==0.8.6,
+             any.base16-bytestring ==1.0.2.0,
              any.base64 ==0.4.2.4,
              any.base64-bytestring ==1.2.1.0,
              any.basement ==0.0.14,
@@ -70,6 +73,8 @@ constraints: any.Cabal ==3.6.3.0,
              cmdargs +quotation -testprog,
              any.colour ==2.3.6,
              any.colourista ==0.1.0.1,
+             any.commonmark ==0.2.2,
+             any.commonmark-extensions ==0.2.3.2,
              any.comonad ==5.0.8,
              comonad +containers +distributive +indexed-traversable,
              any.concurrent-output ==1.10.16,
@@ -81,11 +86,15 @@ constraints: any.Cabal ==3.6.3.0,
              any.contravariant ==1.5.5,
              contravariant +semigroups +statevar +tagged,
              any.cookie ==0.4.5,
+             any.crypto-api ==0.13.3,
+             crypto-api -all_cpolys,
+             any.crypto-pubkey-types ==0.4.3,
              any.cryptohash ==0.11.9,
              any.cryptohash-md5 ==0.11.101.0,
              any.cryptohash-sha1 ==0.11.101.0,
              any.cryptonite ==0.30,
              cryptonite -check_alignment +integer-gmp -old_toolchain_inliner +support_aesni +support_deepseq -support_pclmuldq +support_rdrand -support_sse +use_target_attributes,
+             any.daemons ==0.3.0,
              any.data-default ==0.7.1.1,
              any.data-default-class ==0.1.2.0,
              any.data-default-instances-containers ==0.0.1,
@@ -102,6 +111,8 @@ constraints: any.Cabal ==3.6.3.0,
              any.dlist ==1.0,
              dlist -werror,
              any.easy-file ==0.2.2,
+             any.either ==5.0.2,
+             any.emojis ==0.1.2,
              any.entropy ==0.4.1.7,
              entropy -halvm,
              any.envparse ==0.5.0,
@@ -113,7 +124,11 @@ constraints: any.Cabal ==3.6.3.0,
              any.filtrable ==0.1.6.0,
              filtrable +containers,
              flora -prod,
+             any.foreign-store ==0.2,
              any.free ==5.1.8,
+             any.friendly-time ==0.4.1,
+             any.generic-deriving ==1.14.1,
+             generic-deriving +base-4-9,
              any.ghc ==8.10.7,
              any.ghc-boot ==8.10.7,
              any.ghc-boot-th ==8.10.7,
@@ -125,6 +140,7 @@ constraints: any.Cabal ==3.6.3.0,
              hashable +containers +integer-gmp -random-initial-seed,
              any.haskell-lexer ==1.1,
              any.hedgehog ==1.1.1,
+             any.hostname ==1.0,
              any.hourglass ==0.2.12,
              any.hpc ==0.6.1.0,
              any.hsc2hs ==0.68.8,
@@ -150,6 +166,9 @@ constraints: any.Cabal ==3.6.3.0,
              any.invariant ==0.5.6,
              any.iproute ==1.7.12,
              any.kan-extensions ==5.2.4,
+             any.lens ==5.1.1,
+             lens -benchmark-uniplate -dump-splices +inlining -j +test-hunit +test-properties +test-templates +trustworthy,
+             any.lens-aeson ==1.2.1,
              any.lifted-async ==0.10.2.2,
              any.lifted-base ==0.2.3.12,
              any.log-base ==0.11.1.0,
@@ -165,6 +184,10 @@ constraints: any.Cabal ==3.6.3.0,
              any.mime-types ==0.1.0.9,
              any.mmorph ==1.2.0,
              any.monad-control ==1.0.3.1,
+             any.monad-logger ==0.3.36,
+             monad-logger +template_haskell,
+             any.monad-loops ==0.4.3,
+             monad-loops +base4,
              any.monad-time ==0.3.1.0,
              any.mono-traversable ==1.0.15.3,
              any.mtl ==2.2.2,
@@ -174,12 +197,14 @@ constraints: any.Cabal ==3.6.3.0,
              any.network-byte-order ==0.1.6,
              any.network-info ==0.2.1,
              any.network-uri ==2.6.4.1,
+             any.odd-jobs ==0.2.2,
              any.old-locale ==1.0.0.7,
              any.old-time ==1.1.0.3,
              any.optics-core ==0.4.1,
              optics-core -explicit-generic-labels,
              any.optparse-applicative ==0.17.0.0,
              optparse-applicative +process,
+             any.parallel ==3.2.2.0,
              any.parsec ==3.1.14.0,
              any.password ==3.0.1.0,
              any.password-types ==1.0.0.0,
@@ -188,6 +213,7 @@ constraints: any.Cabal ==3.6.3.0,
              any.pg-entity ==0.0.1.0,
              pg-entity -prod,
              any.pg-transact ==0.3.2.0,
+             any.pipes ==4.3.16,
              any.postgresql-libpq ==0.9.4.3,
              postgresql-libpq -use-pkg-config,
              any.postgresql-migration ==0.2.1.3,
@@ -208,6 +234,8 @@ constraints: any.Cabal ==3.6.3.0,
              any.psqueues ==0.2.7.3,
              any.random ==1.2.1.1,
              any.raven-haskell ==0.1.4.1,
+             any.reflection ==2.1.6,
+             reflection -slow +template-haskell,
              any.regex-applicative ==0.3.4,
              any.resource-pool ==0.3.0.0,
              any.resourcet ==1.2.5,
@@ -223,10 +251,13 @@ constraints: any.Cabal ==3.6.3.0,
              any.semigroups ==0.20,
              semigroups +binary +bytestring -bytestring-builder +containers +deepseq +hashable +tagged +template-haskell +text +transformers +unordered-containers,
              any.servant ==0.19,
+             any.servant-blaze ==0.9.1,
              any.servant-client ==0.19,
              any.servant-client-core ==0.19,
              any.servant-lucid ==0.9.0.5,
              any.servant-server ==0.19.1,
+             any.servant-static-th ==1.0.0.0,
+             servant-static-th -buildexample,
              any.servant-websockets ==2.0.0,
              any.simple-sendfile ==0.2.30,
              simple-sendfile +allow-bsd,
@@ -241,11 +272,14 @@ constraints: any.Cabal ==3.6.3.0,
              any.splitmix ==0.1.0.4,
              splitmix -optimised-mixer,
              any.stm ==2.5.0.1,
+             any.stm-chans ==3.0.0.6,
              any.streaming ==0.2.3.1,
              any.streaming-commons ==0.2.2.4,
              streaming-commons -use-bytestring-builder,
              any.strict ==0.4.0.1,
              strict +assoc,
+             any.string-conv ==0.2.0,
+             string-conv -lib-werror,
              any.string-conversions ==0.4.0.1,
              any.tagged ==0.8.6.1,
              tagged +deepseq +transformers,
@@ -257,6 +291,7 @@ constraints: any.Cabal ==3.6.3.0,
              any.terminal-size ==0.3.3,
              any.terminfo ==0.4.1.4,
              any.text ==1.2.4.1,
+             any.text-conversions ==0.3.1.1,
              any.text-display ==0.0.2.0,
              any.text-manipulate ==0.3.1.0,
              any.text-short ==0.1.5,
@@ -268,7 +303,10 @@ constraints: any.Cabal ==3.6.3.0,
              any.time ==1.9.3,
              any.time-compat ==1.9.6.1,
              time-compat -old-locale,
+             any.time-locale-compat ==0.1.1.5,
+             time-locale-compat +old-locale,
              any.time-manager ==0.0.0,
+             any.timing-convenience ==0.1,
              any.tls ==1.6.0,
              tls +compat -hans +network,
              any.transformers ==0.5.6.2,
@@ -324,6 +362,8 @@ constraints: any.Cabal ==3.6.3.0,
              any.witherable ==0.4.2,
              any.wl-pprint-annotated ==0.1.0.1,
              any.word8 ==0.1.3,
+             any.wreq ==0.5.3.3,
+             wreq -aws -developer +doctest -httpbin,
              any.x509 ==1.7.7,
              any.x509-store ==1.6.9,
              any.x509-system ==1.6.7,

diff --git a/environment.ci.sh b/environment.ci.sh
@@ -2,7 +2,7 @@ source environment.sh
 
 export FLORA_DB_DATABASE="flora_test"
 export FLORA_DB_PASSWORD="postgres"
-export FLORA_DB_POOL_CONNECTIONS=10
+export FLORA_DB_POOL_CONNECTIONS=20
 export FLORA_DB_PORT=5432
 export FLORA_DB_HOST="localhost"
 export FLORA_DB_SUB_POOLS=10

diff --git a/flora.cabal b/flora.cabal
@@ -116,6 +116,7 @@ library
     Flora.Model.User.Query
     Flora.Model.User.Update
     Flora.OddJobs
+    Flora.OddJobs.Types
     Flora.Publish
     Flora.Search
     Flora.ThirdParties.Hackage.API
@@ -186,6 +187,7 @@ library
     , cmark-gfm                  ^>=0.2
     , colourista                 ^>=0.1
     , commonmark
+    , commonmark-extensions
     , containers                 ^>=0.6
     , cookie                     ^>=0.4
     , cryptohash                 ^>=0.11
@@ -195,6 +197,7 @@ library
     , filepath                   ^>=1.4
     , http-api-data              ^>=0.4
     , http-client                ==0.7.10
+    , http-client-tls
     , http-types                 ^>=0.12
     , lens
     , log-base                   ^>=0.11

diff --git a/src/Flora/Environment/OddJobs.hs b/src/Flora/Environment/OddJobs.hs
@@ -1,57 +1,32 @@
-{-# OPTIONS_GHC -Wno-orphans #-}
+{-# LANGUAGE GADTs #-}
 
 -- | Hook oddjobs into flora
-module Flora.Environment.OddJobs
-  ( makeUIConfig
-  , makeConfig
-  )
-where
+module Flora.Environment.OddJobs where
 
-import Data.Aeson
 import Data.Pool hiding (PoolConfig)
-import Data.Text (Text)
-import qualified Data.Text as Text
 import qualified Database.PostgreSQL.Simple as PG
-import Flora.Environment.Config
-import Flora.OddJobs
-import FloraWeb.Server.Logging
 import Log hiding (LogLevel)
 import OddJobs.ConfigBuilder
-import OddJobs.Job
-import OddJobs.Types
-
--- proly should upstream these,
--- kinda dumb the "support" structured logging without the most
--- common method being used
-deriving instance ToJSON FailureMode
-deriving instance ToJSON Job
-instance ToJSON LogEvent where
-  toJSON = \case
-    LogJobStart job -> toJSON ("start" :: Text, job)
-    LogJobSuccess job time -> toJSON ("success" :: Text, job, time)
-    LogJobFailed job exception failuremode finishTime -> toJSON ("failed" :: Text, show exception, job, failuremode, finishTime)
-    LogJobTimeout job -> toJSON ("timed-out" :: Text, job)
-    LogPoll -> toJSON ("poll" :: Text)
-    LogWebUIRequest -> toJSON ("web-ui-request" :: Text)
-    LogText other -> toJSON ("other" :: Text, other)
+import OddJobs.Job (Config (..))
+import OddJobs.Types (ConcurrencyControl (..), Job, UIConfig (..))
 
-structuredLogging :: FloraConfig -> Logger -> LogEvent -> LogLevel -> IO ()
-structuredLogging FloraConfig{..} logger b =
-  runLog environment logger . localDomain "odd-jobs" . \case
-    LevelDebug -> logTrace "LevelDebug" b
-    LevelInfo -> logInfo "LevelInfo" b
-    LevelWarn -> logAttention "LevelWarn" b
-    LevelError -> logAttention "LevelError" b
-    (LevelOther x) -> logAttention ("LevelOther " <> Text.pack (show x)) b
+import Flora.Environment.Config
+import Flora.OddJobs.Types
 
-makeConfig :: FloraConfig -> Logger -> Pool PG.Connection -> Config
-makeConfig cfg@FloraConfig{..} logger pool =
+makeConfig ::
+  JobsRunnerEnv ->
+  FloraConfig ->
+  Logger ->
+  Pool PG.Connection ->
+  (Pool PG.Connection -> Job -> JobsRunnerM ()) ->
+  Config
+makeConfig runnerEnv cfg logger pool runnerContinuation =
   mkConfig
     (flip $ structuredLogging cfg logger)
     jobTableName
     pool
     (MaxConcurrentJobs 1)
-    (runLog environment logger . runner pool)
+    (runJobRunnerM runnerEnv logger . runnerContinuation pool)
     (\x -> x{cfgDeleteSuccessfulJobs = False, cfgDefaultMaxAttempts = 3})
 
 makeUIConfig :: FloraConfig -> Logger -> Pool PG.Connection -> UIConfig

diff --git a/src/Flora/Import/Package.hs b/src/Flora/Import/Package.hs
@@ -44,7 +44,7 @@ import qualified Flora.Import.Categories.Tuning as Tuning
 import qualified Flora.Model.Category.Update as Update
 import Flora.Model.Package.Component as Component
 import Flora.Model.Package.Orphans ()
-import Flora.Model.Package.Types hiding (unPackageName)
+import Flora.Model.Package.Types
 import qualified Flora.Model.Package.Update as Update
 import Flora.Model.Release
 import qualified Flora.Model.Release.Update as Update

diff --git a/src/Flora/Model/Package/Types.hs b/src/Flora/Model/Package/Types.hs
@@ -53,9 +53,6 @@ newtype PackageName = PackageName Text
     (Eq, Ord, FromJSON, ToJSON, FromField, ToField, ToHtml, ToHttpApiData)
     via Text
 
-unPackageName :: PackageName -> Text
-unPackageName (PackageName txt) = txt
-
 instance Pretty PackageName where
   pretty (PackageName txt) = PP.text $ unpack txt