Skip to content

Commit

Permalink
Add scalafmt plugin
Browse files Browse the repository at this point in the history
  • Loading branch information
dilyand committed Nov 1, 2022
1 parent ec97209 commit 4e8a234
Show file tree
Hide file tree
Showing 39 changed files with 982 additions and 801 deletions.
19 changes: 19 additions & 0 deletions .scalafmt.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
version = 3.6.1
runner.dialect = scala213
style = default
maxColumn = 120
optIn.breakChainOnFirstMethodDot = false
assumeStandardLibraryStripMargin = true
align = most
align.tokens."+" = ["|", "!", "!!", "||", "=>", "=", "->", "<-", "|@|", "//", "/", "+", "%", "%%"]
continuationIndent.defnSite = 2
rewrite.rules = [
AsciiSortImports,
AvoidInfix,
PreferCurlyFors,
RedundantBraces,
RedundantParens,
SortModifiers
]
project.git = true
includeNoParensInSelectChains = true
82 changes: 42 additions & 40 deletions build.sbt
Original file line number Diff line number Diff line change
@@ -1,17 +1,15 @@
import sbt.Keys.crossScalaVersions

/**
* Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved.
*
* This program is licensed to you under the Apache License Version 2.0,
* and you may not use this file except in compliance with the Apache License Version 2.0.
* You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the Apache License Version 2.0 is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.ƒ
* See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
*/
/** Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved.
*
* This program is licensed to you under the Apache License Version 2.0, and you may not use this file except in
* compliance with the Apache License Version 2.0. You may obtain a copy of the Apache License Version 2.0 at
* http://www.apache.org/licenses/LICENSE-2.0.
*
* Unless required by applicable law or agreed to in writing, software distributed under the Apache License Version 2.0
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.ƒ See
* the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
*/

lazy val commonSettings = BuildSettings.commonSettings ++
BuildSettings.sbtSiteSettings ++
Expand All @@ -29,22 +27,24 @@ lazy val core = project
)
.enablePlugins(SiteScaladocPlugin, DockerPlugin, JavaAppPackaging)
.settings(commonSettings)
.settings(libraryDependencies ++= Seq(
Dependencies.Libraries.collectionCompat,
Dependencies.Libraries.analyticsSdk,
Dependencies.Libraries.scalaCheck,
Dependencies.Libraries.scalaCheckCats,
Dependencies.Libraries.badRows,
Dependencies.Libraries.httpClient,
Dependencies.Libraries.snowplowRawEvent,
Dependencies.Libraries.collectorPayload,
Dependencies.Libraries.slf4j,
Dependencies.Libraries.thrift,
// Scala (test only)
Dependencies.Libraries.specs2Scalacheck,
Dependencies.Libraries.specs2,
Dependencies.Libraries.specs2Cats
))
.settings(
libraryDependencies ++= Seq(
Dependencies.Libraries.collectionCompat,
Dependencies.Libraries.analyticsSdk,
Dependencies.Libraries.scalaCheck,
Dependencies.Libraries.scalaCheckCats,
Dependencies.Libraries.badRows,
Dependencies.Libraries.httpClient,
Dependencies.Libraries.snowplowRawEvent,
Dependencies.Libraries.collectorPayload,
Dependencies.Libraries.slf4j,
Dependencies.Libraries.thrift,
// Scala (test only)
Dependencies.Libraries.specs2Scalacheck,
Dependencies.Libraries.specs2,
Dependencies.Libraries.specs2Cats
)
)

lazy val sinks = project
.settings(commonSettings)
Expand All @@ -55,18 +55,20 @@ lazy val sinks = project
)
.settings(BuildSettings.dockerSettings)
.settings(BuildSettings.executableSettings)
.settings(libraryDependencies ++= Seq(
Dependencies.Libraries.decline,
Dependencies.Libraries.circeCore,
Dependencies.Libraries.circeConfig,
Dependencies.Libraries.circeExtras,
Dependencies.Libraries.circeGeneric,
Dependencies.Libraries.circeParser,
Dependencies.Libraries.fs2,
Dependencies.Libraries.fs2file,
Dependencies.Libraries.blobstore,
Dependencies.Libraries.specs2
))
.settings(
libraryDependencies ++= Seq(
Dependencies.Libraries.decline,
Dependencies.Libraries.circeCore,
Dependencies.Libraries.circeConfig,
Dependencies.Libraries.circeExtras,
Dependencies.Libraries.circeGeneric,
Dependencies.Libraries.circeParser,
Dependencies.Libraries.fs2,
Dependencies.Libraries.fs2file,
Dependencies.Libraries.blobstore,
Dependencies.Libraries.specs2
)
)
.dependsOn(core)

lazy val root = project.aggregate(core, sinks)
Original file line number Diff line number Diff line change
Expand Up @@ -16,34 +16,38 @@ import cats.implicits._
import org.scalacheck.Gen
import org.scalacheck.cats.implicits._

/**
* Define the vendor and version of the payload, defined by collector endpoint
*/
/** Define the vendor and version of the payload, defined by collector endpoint
*/
final case class Api(vendor: String, version: String) {
override def toString: String = if (vendor == "com.snowplowanalytics.snowplow" && version == "tp1" || vendor == "i" && version == "") "/i" else s"$vendor/$version"
override def toString: String =
if (vendor == "com.snowplowanalytics.snowplow" && version == "tp1" || vendor == "i" && version == "") "/i"
else s"$vendor/$version"
}

object Api {
private val GenI = Gen.const(Api("i", ""))
private val GenI = Gen.const(Api("i", ""))
private val GenIce = Gen.const(Api("ice", ".png"))

def fixedApis: Gen[Api] = Gen.oneOf(GenI, GenIce)

def genApi(nEvents: Int): Gen[Api] = (nEvents match {
case 0 => (genVendor, genVersion)
case 1 => (Gen.const("com.snowplowanalytics.snowplow"), Gen.oneOf("tp1", "tp2"))
case _ => (Gen.const("com.snowplowanalytics.snowplow"), Gen.const("tp2"))
}).mapN(Api.apply)
def genApi(nEvents: Int): Gen[Api] =
(nEvents match {
case 0 => (genVendor, genVersion)
case 1 => (Gen.const("com.snowplowanalytics.snowplow"), Gen.oneOf("tp1", "tp2"))
case _ => (Gen.const("com.snowplowanalytics.snowplow"), Gen.const("tp2"))
}).mapN(Api.apply)

private def genVendor = for {
venPartsN <- Gen.chooseNum(1, 5)
venNs <- Gen.listOfN(venPartsN, Gen.chooseNum(1, 10))
vendorParts <- Gen.sequence[List[String], String](venNs.map(Gen.stringOfN(_, Gen.alphaNumChar)))
sep <- Gen.oneOf("-", ".", "_", "~", "!", "$", "&", "'", "(", ")", "*", "+", ",", ";", "=", ":", "@", "%")
} yield vendorParts.mkString(sep)
private def genVendor =
for {
venPartsN <- Gen.chooseNum(1, 5)
venNs <- Gen.listOfN(venPartsN, Gen.chooseNum(1, 10))
vendorParts <- Gen.sequence[List[String], String](venNs.map(Gen.stringOfN(_, Gen.alphaNumChar)))
sep <- Gen.oneOf("-", ".", "_", "~", "!", "$", "&", "'", "(", ")", "*", "+", ",", ";", "=", ":", "@", "%")
} yield vendorParts.mkString(sep)

private def genVersion = for {
verN <- Gen.chooseNum(1, 10)
version <- Gen.stringOfN(verN, Gen.alphaNumChar)
} yield version
private def genVersion =
for {
verN <- Gen.chooseNum(1, 10)
version <- Gen.stringOfN(verN, Gen.alphaNumChar)
} yield version
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,30 @@ import org.scalacheck.Gen
import java.util.UUID
import java.time.Instant

/**
* Information *derived* by the collector to be used as meta-data (meta-payload)
* Everything else in [[CollectorPayload]] is directly payload (body and queryparams)
*
* @param timestamp collector_tstamp
* @param ipAddress client's IP address, can be later overwritten by `ip` param in
* `enrichments.Transform`
* @param useragent UA header, can be later overwritten by `ua` param in `entichments.Transform`
* @param refererUri extracted from corresponding HTTP header
* @param headers all headers, including UA and referer URI
* @param userId generated by collector-set third-party cookie
*/
/** Information *derived* by the collector to be used as meta-data (meta-payload) Everything else in
* [[CollectorPayload]] is directly payload (body and queryparams)
*
* @param timestamp
* collector_tstamp
* @param ipAddress
* client's IP address, can be later overwritten by `ip` param in `enrichments.Transform`
* @param useragent
* UA header, can be later overwritten by `ua` param in `entichments.Transform`
* @param refererUri
* extracted from corresponding HTTP header
* @param headers
* all headers, including UA and referer URI
* @param userId
* generated by collector-set third-party cookie
*/
final case class CollectorContext(
timestamp: Instant,
ipAddress: Option[IpAddress],
useragent: Option[String],
refererUri: Option[String],
userId: Option[UUID],
headers: Headers
)
{
timestamp: Instant,
ipAddress: Option[IpAddress],
useragent: Option[String],
refererUri: Option[String],
userId: Option[UUID],
headers: Headers
) {
override def toString: String =
s"""
ts: $timestamp
Expand All @@ -51,10 +54,11 @@ final case class CollectorContext(
}

object CollectorContext {
def gen(now: Instant): Gen[CollectorContext] = for {
ts <- genInstant(now)
hdr <- Headers.gen
ip <- IpAddress.genOpt
uid <- Gen.some(Gen.uuid)
} yield CollectorContext(ts, ip, hdr.ua, hdr.ref, uid, hdr)
def gen(now: Instant): Gen[CollectorContext] =
for {
ts <- genInstant(now)
hdr <- Headers.gen
ip <- IpAddress.genOpt
uid <- Gen.some(Gen.uuid)
} yield CollectorContext(ts, ip, hdr.ua, hdr.ref, uid, hdr)
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,27 +31,27 @@ import java.nio.charset.StandardCharsets
import java.time.Instant

final case class CollectorPayload(
api: Api,
payload: List[Body],
source: Source,
context: CollectorContext
) {
api: Api,
payload: List[Body],
source: Source,
context: CollectorContext
) {
private case class PayloadParts(
querystring: List[NameValuePair],
bodyJson: Option[Json],
contentType: Option[String],
timestamp: Long
) {
querystring: List[NameValuePair],
bodyJson: Option[Json],
contentType: Option[String],
timestamp: Long
) {
def body: Option[String] = bodyJson.map(_.noSpaces)
}

private def encodeValue(value: String) = URLEncoder.encode(value, StandardCharsets.UTF_8.toString)

private[this] lazy val parts: PayloadParts = {
val timestamp: Long = context.timestamp.toEpochMilli
val timestamp: Long = context.timestamp.toEpochMilli
var querystring: List[NameValuePair] = List.empty[NameValuePair]
var contentType: Option[String] = None
var bodyJson: Option[Json] = None
var contentType: Option[String] = None
var bodyJson: Option[Json] = None

if (api.version == "tp1") {
querystring = payload.head.toProto.map(kv => new BasicNameValuePair(kv.getName, encodeValue(kv.getValue)))
Expand All @@ -61,15 +61,21 @@ final case class CollectorPayload(
SelfDescribingData(
PayloadDataSchema.Default,
payload.map(body => body.toPayloadElement).asJson
).asJson)
).asJson
)
}
PayloadParts(querystring, bodyJson, contentType, timestamp)
}

def toThrift: CollectorPayload1 = {
def toThrift: CollectorPayload1 =
// Timestamp must be always set, otherwise long will fallback it to 1970-01-01
new CollectorPayload1(CollectorPayload.IgluUri.toSchemaUri, context.ipAddress.map(_.toString).orNull, parts.timestamp, source.encoding, source.name)
.setHostname(source.hostname.orNull)
new CollectorPayload1(
CollectorPayload.IgluUri.toSchemaUri,
context.ipAddress.map(_.toString).orNull,
parts.timestamp,
source.encoding,
source.name
).setHostname(source.hostname.orNull)
.setQuerystring(
(new URIBuilder).setPath(api.toString).setParameters(parts.querystring.asJava).build().getQuery
)
Expand All @@ -80,9 +86,8 @@ final case class CollectorPayload(
.setNetworkUserId(context.userId.map(_.toString).orNull)
.setHeaders(context.headers.toList.asJava)
.setPath(api.toString);
}

override def toString: String = {
override def toString: String =
s"""
#################################### NEW EVENT ####################################
############ ############ ############ QueryString ############ ############ ##########
Expand All @@ -102,36 +107,36 @@ final case class CollectorPayload(
#########################################################################################
""".stripMargin


}

def toRaw: Array[Byte] =
CollectorPayload.serializer.serialize(toThrift)
}

object CollectorPayload {
def genDup(natProb: Float,
synProb: Float,
natTotal: Int,
synTotal: Int,
eventPerPayloadMin: Int,
eventPerPayloadMax: Int,
now: Instant): Gen[CollectorPayload] =
def genDup(
natProb: Float,
synProb: Float,
natTotal: Int,
synTotal: Int,
eventPerPayloadMin: Int,
eventPerPayloadMax: Int,
now: Instant
): Gen[CollectorPayload] =
genWithBody(eventPerPayloadMin, eventPerPayloadMax, Body.genDup(natProb, synProb, natTotal, synTotal, now), now)


private def genWithBody(eventPerPayloadMin: Int, eventPerPayloadMax: Int, bodyGen: Gen[Body], now: Instant) = for {
n <- Gen.chooseNum(eventPerPayloadMin, eventPerPayloadMax)
api <- Api.genApi(n)
src <- Source.gen
cc <- CollectorContext.gen(now)
payload <- Gen.listOfN(n, bodyGen)
} yield CollectorPayload(api, payload, src, cc)
private def genWithBody(eventPerPayloadMin: Int, eventPerPayloadMax: Int, bodyGen: Gen[Body], now: Instant) =
for {
n <- Gen.chooseNum(eventPerPayloadMin, eventPerPayloadMax)
api <- Api.genApi(n)
src <- Source.gen
cc <- CollectorContext.gen(now)
payload <- Gen.listOfN(n, bodyGen)
} yield CollectorPayload(api, payload, src, cc)

def gen(eventPerPayloadMin: Int, eventPerPayloadMax: Int, now: Instant): Gen[CollectorPayload] =
genWithBody(eventPerPayloadMin, eventPerPayloadMax, Body.gen(now), now)

val IgluUri: SchemaKey = SchemaKey("com.snowplowanalytics.snowplow", "CollectorPayload", "thrift", SchemaVer.Full(1, 0, 0))
val IgluUri: SchemaKey =
SchemaKey("com.snowplowanalytics.snowplow", "CollectorPayload", "thrift", SchemaVer.Full(1, 0, 0))

lazy val serializer = new TSerializer()
}
Loading

0 comments on commit 4e8a234

Please sign in to comment.